环境
python=3.11
运行
xinference-local --host 0.0.0.0 --port 8081
bash
nohup xinference-local --host 0.0.0.0 --port 8081 --auth-config ./auth.json > run.log 2>&1 &启动模型
bash
# 登录
xinference login -e http://127.0.0.1:8081 --username aEAIpu2Zgt8q3MPvhnvh --password cG2q0ObqnZP7dn9easZa
# 启动QwQ-32B模型
xinference launch --model-name QwQ-32B --model-type LLM --model-engine vLLM --model-format awq --size-in-billions 32 --quantization Int4 --n-gpu auto --replica 1 --n-worker 1 --reasoning-content false签权
json
{
"auth_config": {
"algorithm": "HS256",
"secret_key": "f000ab9fd88e9bce619ac0acf18256461c9f15d56ce98aa4a0e3a27bb0bd5a95",
"token_expire_in_minutes": 30
},
"user_config": [
{
"username": "aEAIpu2Zgt8q3MPvhnvh",
"password": "cG2q0ObqnZP7dn9easZa",
"permissions": [
"admin"
],
"api_keys": [
"sk-Vyc8z6icvhr5C",
"sk-0Nihu2rZ7Q6z3"
]
},
{
"username": "WfUKpqYpoIdRqQSK9d67",
"password": "MNMlQihF0Ota7xPzu5Th",
"permissions": [
"models:list",
"models:read"
],
"api_keys": [
"sk-vvdwe6xs4BOj9",
"sk-pnjnUTwlXvPXo"
]
}
]
}问题
1. 缺少Cython库
bash
Collecting pynini==2.1.5 (from nemo_text_processing<1.1.0->xinference[all])
Downloading pynini-2.1.5.tar.gz (627 kB)
──────────────────────────────────────── 627.6/627.6 kB 66.5 MB/s eta 0:00:00
Preparing metadata (setup.py) ... error
error: subprocess-exited-with-error
_ python setup.py egg_info did not run successfully.
│ exit code: 1
_─> [6 lines of output]
Traceback (most recent call last):
File "<string>", line 2, in <module>
File "<pip-setuptools-caller>", line 35, in <module>
File "/tmp/pip-install-gagznmaz/pynini_65dfb63d16e64983bc534384e38d95e8/setup.py", line 22, in <module>
from Cython.Build import cythonize
ModuleNotFoundError: No module named 'Cython'
[end of output]
note: This error originates from a subprocess, and is likely not a problem with pip.
error: metadata-generation-failed
_ Encountered error while generating package metadata.
_─> See above for output.
note: This is an issue with the package mentioned above, not pip.
hint: See above for details.解决：
先安装Cython：pip install Cython
再重新：pip install "xinference[all]"
2. numpy 版本错误
bash
from vllm.distributed import get_tensor_model_parallel_rank
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/__init__.py", line 3, in <module>
from .communication_op import *
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/communication_op.py", line 8, in <module>
from .parallel_state import get_tp_group
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/parallel_state.py", line 40, in <module>
import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/kv_transfer/kv_transfer_agent.py", line 16, in <module>
from vllm.distributed.kv_transfer.kv_connector.factory import (
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/kv_transfer/kv_connector/factory.py", line 6, in <module>
from .base import KVConnectorBase
File "/venv/xinference/lib/python3.11/site-packages/vllm/distributed/kv_transfer/kv_connector/base.py", line 15, in <module>
from vllm.sequence import IntermediateTensors
File "/venv/xinference/lib/python3.11/site-packages/vllm/sequence.py", line 17, in <module>
from vllm.inputs import SingletonInputs, SingletonInputsAdapter
File "/venv/xinference/lib/python3.11/site-packages/vllm/inputs/__init__.py", line 9, in <module>
from .registry import (DummyData, InputContext, InputProcessingContext,
File "/venv/xinference/lib/python3.11/site-packages/vllm/inputs/registry.py", line 16, in <module>
from vllm.transformers_utils.tokenizer import AnyTokenizer
File "/venv/xinference/lib/python3.11/site-packages/vllm/transformers_utils/tokenizer.py", line 12, in <module>
from transformers import (AutoTokenizer, PreTrainedTokenizer,
File "<frozen importlib._bootstrap>", line 1229, in _handle_fromlist
File "/venv/xinference/lib/python3.11/site-packages/transformers/utils/import_utils.py", line 1956, in __getattr__
value = getattr(module, name)
^^^^^^^^^^^^^^^^^^^^^
File "/venv/xinference/lib/python3.11/site-packages/transformers/utils/import_utils.py", line 1955, in __getattr__
module = self._get_module(self._class_to_module[name])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/venv/xinference/lib/python3.11/site-packages/transformers/utils/import_utils.py", line 1969, in _get_module
raise RuntimeError(
RuntimeError: Failed to import transformers.models.auto.tokenization_auto because of the following error (look up to see its traceback):
Failed to import transformers.generation.utils because of the following error (look up to see its traceback):
numpy.core.multiarray failed to import**解决：**使用numpy==1.26.0