diff --git a/Dockerfile b/Dockerfile index c148220..b6c3cb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0 +FROM docker.1ms.run/vllm/vllm-openai-rocm:latest WORKDIR /workspace diff --git a/README.md b/README.md index 4320fa6..389f051 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # ROCm vLLM 容器化推理项目 -基于镜像 `rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0` 的 Python 推理服务,适配双 AMD R9700 32G GPU。 +基于镜像 `docker.1ms.run/vllm/vllm-openai-rocm:latest` 的 Python + vLLM 推理服务,适配双 AMD R9700 32G GPU。 ## 项目目标 - 提供可容器化部署的模型推理 API -- 使用 vLLM 在双 GPU 上进行张量并行推理 +- 使用 vLLM + ROCm 在 AMD GPU 上执行推理 - 提供健康检查、鉴权和参数化配置能力 - 暴露 `8001` OpenAI 标准协议接口,兼容 OpenClaw 调用 @@ -72,7 +72,7 @@ - `models.default`:默认模型名 - `models.selected`:当前生效模型名 - `models.profiles`:模型配置集合 -- 每个模型必须包含:`local_path`、`ctx`、`max_num_seqs`、`max_tokens`、`gpu_util`、`valid_tp` +- 每个模型必须包含:`local_path`,并建议补充 `ctx`、`max_num_seqs`、`max_tokens` 启动时会按以下优先级选模型: @@ -146,15 +146,6 @@ curl -X POST "http://localhost:8001/v1/chat/completions" \ ## 常见故障排查 -- 报错 `model type qwen3_5_moe ... Transformers does not recognize this architecture` 时,先重建镜像以更新依赖: - -```bash -docker compose build --no-cache -docker compose up -d --force-recreate -``` - -- 如果看到 `No services to build`,说明你执行的是 `docker compose up -d --force-recreate` 且未触发重建;必须先单独执行 `docker compose build --no-cache`。 - -- 构建阶段若出现 `git clone https://github.com/huggingface/transformers.git` 失败,请改用 PyPI 版本依赖(项目已默认 `transformers>=4.57.0,<5`),避免源码拉取中断导致构建失败。 - -- 若仍报相同错误,说明当前 `vLLM/Transformers` 组合不支持该模型架构,建议在 `config.json` 切到其他本地模型,或升级基础镜像到更新的 vLLM 版本。 +- 报错 `model type ... Transformers does not recognize this architecture` 时,说明当前模型与镜像内依赖不兼容,建议更换模型或升级镜像版本。 +- 若模型目录存在但仍加载失败,检查挂载路径是否为 `/opt/model:/opt/model:ro`,并确认容器内可见模型文件。 +- 如果看到 `No services to build`,说明未触发重建;需要先执行 `docker compose build --no-cache` 再 `up`。 diff --git a/app/engine.py b/app/engine.py index 416f438..b18ca09 100644 --- a/app/engine.py +++ b/app/engine.py @@ -22,6 +22,9 @@ class InferenceEngine: revision=settings.revision, ) + def close(self) -> None: + return None + def generate(self, req: GenerateRequest) -> GenerateResponse: sampling_params = SamplingParams( temperature=req.temperature, diff --git a/app/main.py b/app/main.py index 74e86f1..160b1e9 100644 --- a/app/main.py +++ b/app/main.py @@ -25,6 +25,8 @@ async def lifespan(_: FastAPI): settings = get_settings() engine = InferenceEngine(settings) yield + if engine is not None: + engine.close() engine = None diff --git a/app/model_catalog.py b/app/model_catalog.py index b4097fc..f5c4e15 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -43,11 +43,12 @@ def _resolve_profile_model_path(profile: dict[str, Any], model_root: str, model_ raise ValueError(f"model profile '{model_key}' must provide local_path") if "://" in local_path: raise ValueError(f"model profile '{model_key}' local_path must be local filesystem path") - if local_path.startswith("/"): - return local_path - if not model_root: - raise ValueError("config.json model_root cannot be empty when local_path is relative") - return _join_posix(model_root, local_path) + resolved = local_path + if not local_path.startswith("/"): + if not model_root: + raise ValueError("config.json model_root cannot be empty when local_path is relative") + resolved = _join_posix(model_root, local_path) + return resolved def load_catalog(catalog_path: str = "config.json") -> dict[str, Any]: diff --git a/app/start_openai.py b/app/start_openai.py index 9e24f9f..b6b13d5 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -18,9 +18,9 @@ def build_command() -> list[str]: os.environ[key] = value host = str(runtime["openai_host"]) port = str(runtime["openai_port"]) + api_key = runtime["api_key"] or "" dtype = str(runtime["dtype"]) revision = runtime["revision"] or "" - api_key = runtime["api_key"] or "" cmd = [ sys.executable, "-m", diff --git a/requirements.txt b/requirements.txt index e52d3f5..3ae98d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ fastapi==0.116.1 uvicorn==0.35.0 pydantic==2.11.7 -transformers>=4.57.0,<5