x

2026-03-29 05:22:25 +08:00
parent 6790945334
commit 4db5de796a
5 changed files with 129 additions and 2 deletions
@@ -3,6 +3,8 @@ MODEL_KEY=Qwen3.5-35B-A3B-GPTQ-Int4
 MODEL_NAME=
 HOST=0.0.0.0
 PORT=8000
+OPENAI_HOST=0.0.0.0
+OPENAI_PORT=8001
 TENSOR_PARALLEL_SIZE=2
 GPU_MEMORY_UTILIZATION=0.92
 MAX_MODEL_LEN=8192
@@ -18,6 +18,6 @@ ENV DTYPE=bfloat16
 ENV ENFORCE_EAGER=false
 ENV TRUST_REMOTE_CODE=false

-EXPOSE 8000
+EXPOSE 8000 8001

 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
@@ -7,6 +7,7 @@
 - 提供可容器化部署的模型推理 API
 - 使用 vLLM 在双 GPU 上进行张量并行推理
 - 提供健康检查、鉴权和参数化配置能力
+- 暴露 `8001` OpenAI 标准协议接口，兼容 OpenClaw 调用

 ## 目录结构

@@ -57,6 +58,8 @@
 - `MODEL_CONFIG_FILE`：模型配置文件路径，默认 `config.json`
 - `MODEL_KEY`：要启动的模型键，留空时使用 `config.json` 中 `default_model`
 - `MODEL_NAME`：兜底模型名（仅在不使用配置文件时生效）
+- `OPENAI_HOST`：OpenAI 协议服务监听地址，默认 `0.0.0.0`
+- `OPENAI_PORT`：OpenAI 协议服务监听端口，默认 `8001`
 - `TENSOR_PARALLEL_SIZE`：张量并行数，双卡建议为 `2`
 - `GPU_MEMORY_UTILIZATION`：单卡显存利用率，建议 `0.90~0.95`
 - `MAX_MODEL_LEN`：模型最大上下文长度
@@ -104,12 +107,18 @@ cp .env.example .env
 docker compose up -d --build
 ```

-4. 验证服务：
+4. 验证自定义推理服务：

 ```bash
 curl http://localhost:8000/health
 ```

+5. 验证 OpenAI 协议服务：
+
+```bash
+curl http://localhost:8001/v1/models
+```
+
 ## 推理请求示例

 ```bash
@@ -118,6 +127,22 @@ curl -X POST "http://localhost:8000/v1/generate" \
  -d "{\"prompt\":\"请用三句话介绍大模型推理优化\",\"max_tokens\":128,\"temperature\":0.7}"
 ```

+## OpenAI 协议示例（8001）
+
+```bash
+curl -X POST "http://localhost:8001/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer ${API_KEY}" \
+  -d "{\"model\":\"Qwen3.5-35B-A3B-GPTQ-Int4\",\"messages\":[{\"role\":\"user\",\"content\":\"你好，介绍一下你自己\"}],\"temperature\":0.7}"
+```
+
+## OpenClaw 调用说明
+
+- Base URL 使用 `http://<服务器IP>:8001/v1`
+- API Key 使用 `.env` 中 `API_KEY` 对应值
+- 模型名使用 `config.json` 中 `served_model_name`
+- 若使用工具调用，`config.json` 中应配置 `tool_call_parser` 与 `enable_auto_tool_choice`
+
 ## 双 AMD R9700 调优建议

 - 首选 `TENSOR_PARALLEL_SIZE=2`
@@ -0,0 +1,68 @@
+import os
+import subprocess
+import sys
+
+from app.model_catalog import resolve_model_profile
+
+
+def build_command() -> list[str]:
+    config_file = os.getenv("MODEL_CONFIG_FILE", "config.json")
+    model_key = os.getenv("MODEL_KEY")
+    requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2"))
+    _, updates, env_vars = resolve_model_profile(
+        catalog_path=config_file,
+        requested_model=model_key,
+        requested_tp=requested_tp,
+    )
+    for key, value in env_vars.items():
+        os.environ[key] = value
+    host = os.getenv("OPENAI_HOST", "0.0.0.0")
+    port = os.getenv("OPENAI_PORT", "8001")
+    dtype = os.getenv("DTYPE", "bfloat16")
+    revision = os.getenv("REVISION", "").strip()
+    api_key = os.getenv("API_KEY", "").strip()
+    cmd = [
+        sys.executable,
+        "-m",
+        "vllm.entrypoints.openai.api_server",
+        "--host",
+        host,
+        "--port",
+        port,
+        "--model",
+        str(updates["model_name"]),
+        "--served-model-name",
+        str(updates["served_model_name"]),
+        "--tensor-parallel-size",
+        str(updates["tensor_parallel_size"]),
+        "--max-model-len",
+        str(updates["max_model_len"]),
+        "--gpu-memory-utilization",
+        str(updates["gpu_memory_utilization"]),
+        "--max-num-seqs",
+        str(updates["max_num_seqs"]),
+        "--dtype",
+        dtype,
+    ]
+    if updates["trust_remote_code"]:
+        cmd.append("--trust-remote-code")
+    if updates["enforce_eager"]:
+        cmd.append("--enforce-eager")
+    if updates["enable_auto_tool_choice"]:
+        cmd.append("--enable-auto-tool-choice")
+    if updates["tool_call_parser"]:
+        cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
+    if revision:
+        cmd.extend(["--revision", revision])
+    if api_key:
+        cmd.extend(["--api-key", api_key])
+    return cmd
+
+
+def main() -> None:
+    command = build_command()
+    raise SystemExit(subprocess.call(command))
+
+
+if __name__ == "__main__":
+    main()
@@ -9,6 +9,38 @@ services:
      - "8000:8000"
    env_file:
      - .env
+    volumes:
+      - /opt/model:/opt/model:ro
+      - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro
+    devices:
+      - /dev/kfd
+      - /dev/dri
+    group_add:
+      - video
+      - render
+    ipc: host
+    shm_size: 16g
+    cap_add:
+      - SYS_PTRACE
+    security_opt:
+      - seccomp=unconfined
+    restart: unless-stopped
+  openai_api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: rocm-vllm-inference:latest
+    container_name: rocm-vllm-openai
+    command: ["python", "-m", "app.start_openai"]
+    ports:
+      - "8001:8001"
+    env_file:
+      - .env
+    environment:
+      OPENAI_PORT: 8001
+    volumes:
+      - /opt/model:/opt/model:ro
+      - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro
    devices:
      - /dev/kfd
      - /dev/dri