5种搭建LLM服务的方法和代码示例

 pip install llama-cpp-python[server] \
   --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu

 mkdir -p models/7B
 wget -O models/7B/llama-2-7b-chat.Q5_K_M.gguf https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf?download=true

 python3 -m llama_cpp.server --model models/7B/llama-2-7b-chat.Q5_K_M.gguf

 messages=[
     {"role": "system", "content": "You are a helpful assistant."},
     {
         "role": "user",
         "content": "What are the names of the four main characters of South Park?",
     },
 ]

 pip install vllm

 python -m vllm.entrypoints.openai.api_server --model TheBloke/Llama-2-7B-Chat-AWQ --api-key DEFAULT --quantization awq --enforce-eager

 sudo apt install nvidia-cuda-toolkit

 distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
 curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
 curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
 
 sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
 sudo systemctl restart docker

 sudo tee /etc/docker/daemon.json <<EOF
 {
     "runtimes": {
         "nvidia": {
             "path": "/usr/bin/nvidia-container-runtime",
             "runtimeArgs": []
         }
     }
 }
 EOF
 sudo pkill -SIGHUP dockerd

 docker run --runtime nvidia --gpus all \
     -v ~/.cache/huggingface:/root/.cache/huggingface \
     -p 8000:8000 \
     --ipc=host \
     vllm/vllm-openai:latest \
     --model TheBloke/Llama-2-7B-Chat-AWQ \
     --quantization awq --enforce-eager

 pip install modal

 modal setup

  # https://modal.com/docs/examples/vllm_mixtral
 import os 
 import time
 
 from modal import Image, Stub, enter, exit, gpu, method
 
 APP_NAME = "example-vllm-llama-chat"
 MODEL_DIR = "/model"
 BASE_MODEL = "TheBloke/Llama-2-7B-Chat-AWQ"
 GPU_CONFIG = gpu.T4(count=1)

 vllm_image = ( # https://modal.com/docs/examples/vllm_mixtral
     Image.from_registry("nvidia/cuda:12.1.1-devel-ubuntu22.04", add_python="3.10")
     .pip_install(
         "vllm==0.3.2",
         "huggingface_hub==0.19.4",
         "hf-transfer==0.1.4",
         "torch==2.1.2",
     )
     .env({"HF_HUB_ENABLE_HF_TRANSFER": "1"})
     .run_function(download_model_to_folder, timeout=60 * 20)
 )

 stub = Stub(APP_NAME)

 class Model:  
     @enter()  # Lifecycle functions
     def start_engine(self):
         import time
 
         from vllm.engine.arg_utils import AsyncEngineArgs
         from vllm.engine.async_llm_engine import AsyncLLMEngine
 
         print("🥶 cold starting inference")
         start = time.monotonic_ns()
 
         engine_args = AsyncEngineArgs(
             model=MODEL_DIR,
             tensor_parallel_size=GPU_CONFIG.count,
             gpu_memory_utilization=0.90,
             enforce_eager=False,  # capture the graph for faster inference, but slower cold starts
             disable_log_stats=True,  # disable logging so we can stream tokens

 @stub.function()
 def generate(user_question: str):
     model = Model()
 
     print("Sending new request:", user_question, "\n\n")
 
     result = ""
     for text in model.completion_stream.remote_gen(user_question):
         print(text, end="", flush=True)
         result += text
 
     return result

 modal deploy vllm_modal_deploy.py

 import timeit
 
 import modal
 
 APP_NAME = "example-vllm-llama-chat"
 f = modal.Function.lookup(APP_NAME, "generate")
 
 start_time = timeit.default_timer()
 
 print(f.remote("What are the names of the four main characters of South park?"))
 
 elapsed = timeit.default_timer() - start_time
 
 print(f"{elapsed=}")

 export API_KEY="CHANGEME"
 export MODEL="meta-llama/Llama-2-7b-chat-hf"
 export BASE_URL="https://api.endpoints.anyscale.com/v1"
 python openai_client.py