vllm+vllm-ascend本地部署QwQ-32B

https://mirrors.tools.huawei.com/mirrorDetail/67b75986118b030fb5934fc7?mirrorName=huggingface&catalog=llms

docker run -itd --net=host --name vllm-ascend-QwQ-32B --device /dev/davinci0 --device /dev/davinci1 --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc -v /usr/local/dcmi:/usr/local/dcmi -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info -v /etc/ascend_install.info:/etc/ascend_install.info -v /xxx/models/llmmodels:/usr1/project/models quay.io/ascend/vllm-ascend:v0.7.3-dev bash

# Install vLLM
git clone --depth 1 --branch v0.8.4 https://github.com/vllm-project/vllm
cd vllm
VLLM_TARGET_DEVICE=empty pip install . --extra-index https://download.pytorch.org/whl/cpu/
cd ..

# Install vLLM Ascend
git clone  --depth 1 --branch v0.8.4rc1 https://github.com/vllm-project/vllm-ascend.git
cd vllm-ascend
pip install -e . --extra-index https://download.pytorch.org/whl/cpu/
cd ..

vllm serve /usr1/project/models/QwQ-32B --tensor_parallel_size 2 --served-model-name "QwQ-32B" --max-num-seqs 256 --max-model-len=4096 --host xx.xx.xx.xx --port 8001 &