diff --git a/Dockerfile b/Dockerfile index 4792cbd..fa70850 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0 +# FROM docker.1ms.run/vllm/vllm-openai-rocm:latest +FROM vllm/vllm-openai-rocm:nightly WORKDIR /workspace diff --git a/README.md b/README.md index c21c830..2dfc4c7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ROCm vLLM 容器化推理项目 -基于镜像 `rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0` 的 Python + vLLM 推理服务,适配双 AMD R9700 32G GPU。 +基于镜像 `docker.1ms.run/vllm/vllm-openai-rocm:latest` 的 Python + vLLM 推理服务,适配双 AMD R9700 32G GPU。 ## 项目目标 @@ -68,7 +68,7 @@ 1. 预拉取基础镜像(与官方文档一致): ```bash -docker pull rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0 +docker pull docker.1ms.run/vllm/vllm-openai-rocm:latest ``` 2. 修改 `config.json` 中的 `models.selected` 与服务参数。 @@ -124,7 +124,7 @@ curl -X POST "http://localhost:8001/v1/chat/completions" \ ## 常见故障排查 -- 报错 `model type ... Transformers does not recognize this architecture` 时,先确认已使用当前 Dockerfile 重建镜像(其中会升级 `transformers`),并执行 `docker compose build --no-cache` 后再启动。 +- 报错 `Model architectures ['Qwen3_5MoeForConditionalGeneration'] are not supported for now` 时,在对应 profile 增加 `model_impl=transformers`,并使用当前 Dockerfile 重建镜像后再启动。 - 报错 `model config (gptq) does not match quantization argument (gptq_marlin)` 时,将该模型配置改为 `dtype=float16` 且 `quantization=gptq`。 - 报错 `RPC call to sample_tokens timed out` 或出现 `GPU core dump` 时,先下调模型配置为更稳参数:`ctx=32768`、`max_num_seqs=4`、`max_tokens=2048`、`gpu_util=0.90`,并开启 `enforce_eager=true`。 - 若模型目录存在但仍加载失败,检查挂载路径是否为 `/opt/model:/opt/model:ro`,并确认容器内可见模型文件。 diff --git a/app/model_catalog.py b/app/model_catalog.py index bdee6c4..eb76872 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -110,6 +110,7 @@ def resolve_model_profile( "served_model_name": profile.get("served_model_name", model_key), "dtype": _to_str(profile.get("dtype")), "quantization": _to_str(profile.get("quantization")), + "model_impl": _to_str(profile.get("model_impl")), "reasoning_parser": _to_str(profile.get("reasoning_parser")), "max_model_len": _to_int(profile.get("ctx"), 8192), "max_num_seqs": _to_int(profile.get("max_num_seqs"), 64), diff --git a/app/start_openai.py b/app/start_openai.py index 7dc3f7a..7f1caf3 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -25,6 +25,7 @@ def build_command() -> list[str]: api_key = runtime["api_key"] or "" dtype = str(updates["dtype"] or runtime["dtype"]) quantization = str(updates["quantization"] or "").strip() + model_impl = str(updates["model_impl"] or "").strip() reasoning_parser = str(updates["reasoning_parser"] or "").strip() revision = runtime["revision"] or "" cmd = [ @@ -68,6 +69,8 @@ def build_command() -> list[str]: cmd.extend(["--reasoning-parser", reasoning_parser]) if quantization: cmd.extend(["--quantization", quantization]) + if model_impl: + cmd.extend(["--model-impl", model_impl]) if revision: cmd.extend(["--revision", revision]) if api_key: diff --git a/config.json b/config.json index 5d0a2ce..1e35fab 100644 --- a/config.json +++ b/config.json @@ -73,6 +73,7 @@ "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4", "dtype": "float16", "quantization": "gptq", + "model_impl": "transformers", "ctx": "32768", "trust_remote": true, "valid_tp": [2],