https://github.com/oobabooga/exllamav2/releases/download/v0.2.4/exllamav2-0.2.4+cu121.torch2.4.1-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.4/exllamav2-0.2.4+cu121.torch2.4.1...
sse-starlette==1.6.5 tiktoken # llama-cpp-python (CUDA) https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.64-cu121/llama_cpp_python-0.2.64-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/abetlen/llama-cpp...
import openai openai.api_key = "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # can be anything openai.api_base = "http://127.0.0.1:8000/v1" openai.Completion.create( model="text-davinci-003", # currently can be anything prompt="The quick brown fox jumps", max_tokens=5, ) ...