x

.env.example

@@ -1,19 +0,0 @@
-MODEL_CONFIG_FILE=config.json
-MODEL_KEY=Qwen3.5-35B-A3B-GPTQ-Int4
-MODEL_NAME=
-HOST=0.0.0.0
-PORT=8000
-OPENAI_HOST=0.0.0.0
-OPENAI_PORT=8001
-TENSOR_PARALLEL_SIZE=2
-GPU_MEMORY_UTILIZATION=0.92
-MAX_MODEL_LEN=8192
-MAX_NUM_SEQS=64
-MAX_TOKENS=4096
-DTYPE=bfloat16
-ENFORCE_EAGER=false
-TRUST_REMOTE_CODE=false
-TOOL_CALL_PARSER=
-ENABLE_AUTO_TOOL_CHOICE=false
-REVISION=
-API_KEY=

Dockerfile

@@ -8,16 +8,7 @@ RUN pip install --no-cache-dir -r /workspace/requirements.txt
 COPY app /workspace/app
 
 ENV PYTHONUNBUFFERED=1
-ENV HOST=0.0.0.0
-ENV PORT=8000
-ENV TENSOR_PARALLEL_SIZE=2
-ENV GPU_MEMORY_UTILIZATION=0.92
-ENV MAX_MODEL_LEN=8192
-ENV MAX_NUM_SEQS=64
-ENV DTYPE=bfloat16
-ENV ENFORCE_EAGER=false
-ENV TRUST_REMOTE_CODE=false
 
 EXPOSE 8000 8001
 
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["python", "-m", "app.start_api"]

README.md

@@ -18,9 +18,10 @@
 │   ├── engine.py
 │   ├── main.py
 │   ├── model_catalog.py
+│   ├── start_api.py
+│   ├── start_openai.py
 │   └── schemas.py
 ├── .dockerignore
-├── .env.example
 ├── config.json
 ├── docker-compose.yml
 ├── Dockerfile
@@ -53,20 +54,14 @@
 
 ## 配置项
 
-可通过 `.env` 配置，建议先复制 `.env.example`：
+项目只读取一个配置文件：`config.json`。
 
-- `MODEL_CONFIG_FILE`：模型配置文件路径，默认 `config.json`
+- `services.api.host` / `services.api.port`：自定义推理服务监听地址与端口（默认 `0.0.0.0:8000`）
-- `MODEL_KEY`：要启动的模型键，留空时使用 `config.json` 中 `default_model`
+- `services.openai.host` / `services.openai.port`：OpenAI 协议服务监听地址与端口（默认 `0.0.0.0:8001`）
-- `MODEL_NAME`：兜底模型名（仅在不使用配置文件时生效）
+- `api_key`：接口访问密钥（同时用于 8000 与 8001）
-- `OPENAI_HOST`：OpenAI 协议服务监听地址，默认 `0.0.0.0`
+- `tensor_parallel_size`：张量并行数，双卡建议 `2`
-- `OPENAI_PORT`：OpenAI 协议服务监听端口，默认 `8001`
+- `dtype`：推理精度，默认 `bfloat16`
-- `TENSOR_PARALLEL_SIZE`：张量并行数，双卡建议为 `2`
+- `selected_model`：当前生效模型，留空时回退到 `default_model`
-- `GPU_MEMORY_UTILIZATION`：单卡显存利用率，建议 `0.90~0.95`
-- `MAX_MODEL_LEN`：模型最大上下文长度
-- `MAX_NUM_SEQS`：并发序列数量
-- `MAX_TOKENS`：单请求最大生成长度
-- `DTYPE`：精度类型，默认 `bfloat16`
-- `API_KEY`：可选接口访问密钥
 
 ## config.json 说明
 
@@ -78,7 +73,7 @@
 
 启动时会按以下优先级选模型：
 
-1. `.env` 中 `MODEL_KEY`
+1. `config.json` 中 `selected_model`
 2. `config.json` 中 `default_model`
 
 模型被选中后，会自动覆盖运行参数，包括：
@@ -93,27 +88,21 @@
 
 ## 部署步骤
 
-1. 准备环境变量：
+1. 修改 `config.json` 中的 `selected_model` 与服务参数。
 
-```bash
+2. 构建并启动容器：
-cp .env.example .env
-```
-
-2. 修改 `.env` 中的 `MODEL_KEY`，选择要启动的模型。
-
-3. 构建并启动容器：
 
 ```bash
 docker compose up -d --build
 ```
 
-4. 验证自定义推理服务：
+3. 验证自定义推理服务：
 
 ```bash
 curl http://localhost:8000/health
 ```
 
-5. 验证 OpenAI 协议服务：
+4. 验证 OpenAI 协议服务：
 
 ```bash
 curl http://localhost:8001/v1/models
@@ -132,14 +121,14 @@ curl -X POST "http://localhost:8000/v1/generate" \
 ```bash
 curl -X POST "http://localhost:8001/v1/chat/completions" \
   -H "Content-Type: application/json" \
-  -H "Authorization: Bearer ${API_KEY}" \
+  -H "Authorization: Bearer <config.json中的api_key>" \
   -d "{\"model\":\"Qwen3.5-35B-A3B-GPTQ-Int4\",\"messages\":[{\"role\":\"user\",\"content\":\"你好，介绍一下你自己\"}],\"temperature\":0.7}"
 ```
 
 ## OpenClaw 调用说明
 
 - Base URL 使用 `http://<服务器IP>:8001/v1`
-- API Key 使用 `.env` 中 `API_KEY` 对应值
+- API Key 使用 `config.json` 中 `api_key`
 - 模型名使用 `config.json` 中 `served_model_name`
 - 若使用工具调用，`config.json` 中应配置 `tool_call_parser` 与 `enable_auto_tool_choice`
 

app/config.py

@@ -2,44 +2,56 @@ from functools import lru_cache
 import os
 from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel
-from pydantic_settings import BaseSettings, SettingsConfigDict
 
-from app.model_catalog import resolve_model_profile
+from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
 
 
-class Settings(BaseSettings):
+class Settings(BaseModel):
-    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
+    config_file: str = "config.json"
-
+    model_key: Optional[str] = None
-    config_file: str = Field(default="config.json", alias="MODEL_CONFIG_FILE")
-    model_key: Optional[str] = Field(default=None, alias="MODEL_KEY")
     selected_model: Optional[str] = None
-    model_name: str = Field(default="", alias="MODEL_NAME")
+    model_name: str = ""
     served_model_name: Optional[str] = None
-    host: str = Field(default="0.0.0.0", alias="HOST")
+    host: str = "0.0.0.0"
-    port: int = Field(default=8000, alias="PORT")
+    port: int = 8000
-    max_model_len: int = Field(default=8192, alias="MAX_MODEL_LEN")
+    openai_host: str = "0.0.0.0"
-    gpu_memory_utilization: float = Field(default=0.92, alias="GPU_MEMORY_UTILIZATION")
+    openai_port: int = 8001
-    tensor_parallel_size: int = Field(default=2, alias="TENSOR_PARALLEL_SIZE")
+    max_model_len: int = 8192
-    max_num_seqs: int = Field(default=64, alias="MAX_NUM_SEQS")
+    gpu_memory_utilization: float = 0.92
-    max_tokens: int = Field(default=4096, alias="MAX_TOKENS")
+    tensor_parallel_size: int = 2
-    dtype: str = Field(default="bfloat16", alias="DTYPE")
+    max_num_seqs: int = 64
-    enforce_eager: bool = Field(default=False, alias="ENFORCE_EAGER")
+    max_tokens: int = 4096
-    trust_remote_code: bool = Field(default=False, alias="TRUST_REMOTE_CODE")
+    dtype: str = "bfloat16"
-    tool_call_parser: Optional[str] = Field(default=None, alias="TOOL_CALL_PARSER")
+    enforce_eager: bool = False
-    enable_auto_tool_choice: bool = Field(default=False, alias="ENABLE_AUTO_TOOL_CHOICE")
+    trust_remote_code: bool = False
-    revision: Optional[str] = Field(default=None, alias="REVISION")
+    tool_call_parser: Optional[str] = None
-    api_key: Optional[str] = Field(default=None, alias="API_KEY")
+    enable_auto_tool_choice: bool = False
+    revision: Optional[str] = None
+    api_key: Optional[str] = None
 
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
-    settings = Settings()
+    catalog = load_catalog("config.json")
+    runtime = resolve_runtime_settings(catalog)
+    settings = Settings(
+        config_file="config.json",
+        model_key=runtime["model_key"],
+        host=runtime["host"],
+        port=runtime["port"],
+        openai_host=runtime["openai_host"],
+        openai_port=runtime["openai_port"],
+        api_key=runtime["api_key"],
+        tensor_parallel_size=runtime["tensor_parallel_size"],
+        dtype=runtime["dtype"],
+        revision=runtime["revision"],
+    )
     _, updates, env_vars = resolve_model_profile(
-        catalog_path=settings.config_file,
+        content=catalog,
         requested_model=settings.model_key,
         requested_tp=settings.tensor_parallel_size,
     )
     for key, value in env_vars.items():
         os.environ[key] = value
-    return settings.model_copy(update=updates)
+    return settings.model_copy(update=updates | runtime)

app/model_catalog.py

@@ -29,14 +29,37 @@ def _to_float(value: Any, default: float) -> float:
         return default
 
 
-def resolve_model_profile(
+def load_catalog(catalog_path: str = "config.json") -> dict[str, Any]:
-    catalog_path: str, requested_model: str | None, requested_tp: int
-) -> tuple[str, dict[str, Any], dict[str, str]]:
     content = json.loads(Path(catalog_path).read_text(encoding="utf-8"))
     if not isinstance(content, dict):
         raise ValueError("config.json must be a JSON object")
-    default_model = content.get("default_model")
+    return content
-    profiles = {k: v for k, v in content.items() if k != "default_model"}
+
+
+def resolve_runtime_settings(content: dict[str, Any]) -> dict[str, Any]:
+    services = content.get("services", {})
+    api_service = dict(services.get("api", {}))
+    openai_service = dict(services.get("openai", {}))
+    models = dict(content.get("models", {}))
+    return {
+        "host": str(api_service.get("host", "0.0.0.0")),
+        "port": _to_int(api_service.get("port"), 8000),
+        "openai_host": str(openai_service.get("host", "0.0.0.0")),
+        "openai_port": _to_int(openai_service.get("port"), 8001),
+        "api_key": str(content.get("api_key", "")).strip() or None,
+        "tensor_parallel_size": _to_int(content.get("tensor_parallel_size"), 2),
+        "dtype": str(content.get("dtype", "bfloat16")),
+        "revision": str(content.get("revision", "")).strip() or None,
+        "model_key": str(models.get("selected", "")).strip() or None,
+    }
+
+
+def resolve_model_profile(
+    content: dict[str, Any], requested_model: str | None, requested_tp: int
+) -> tuple[str, dict[str, Any], dict[str, str]]:
+    models = dict(content.get("models", {}))
+    profiles = dict(models.get("profiles", {}))
+    default_model = models.get("default")
     model_key = requested_model or default_model
     if not model_key or model_key not in profiles:
         raise ValueError(f"model profile '{model_key}' not found in config.json")

app/start_api.py

@@ -0,0 +1,23 @@
+import subprocess
+import sys
+
+from app.config import get_settings
+
+
+def main() -> None:
+    settings = get_settings()
+    command = [
+        sys.executable,
+        "-m",
+        "uvicorn",
+        "app.main:app",
+        "--host",
+        settings.host,
+        "--port",
+        str(settings.port),
+    ]
+    raise SystemExit(subprocess.call(command))
+
+
+if __name__ == "__main__":
+    main()

app/start_openai.py

@@ -2,25 +2,25 @@ import os
 import subprocess
 import sys
 
-from app.model_catalog import resolve_model_profile
+from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
 
 
 def build_command() -> list[str]:
-    config_file = os.getenv("MODEL_CONFIG_FILE", "config.json")
+    config_file = "config.json"
-    model_key = os.getenv("MODEL_KEY")
+    catalog = load_catalog(config_file)
-    requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2"))
+    runtime = resolve_runtime_settings(catalog)
     _, updates, env_vars = resolve_model_profile(
-        catalog_path=config_file,
+        content=catalog,
-        requested_model=model_key,
+        requested_model=runtime["model_key"],
-        requested_tp=requested_tp,
+        requested_tp=runtime["tensor_parallel_size"],
     )
     for key, value in env_vars.items():
         os.environ[key] = value
-    host = os.getenv("OPENAI_HOST", "0.0.0.0")
+    host = str(runtime["openai_host"])
-    port = os.getenv("OPENAI_PORT", "8001")
+    port = str(runtime["openai_port"])
-    dtype = os.getenv("DTYPE", "bfloat16")
+    dtype = str(runtime["dtype"])
-    revision = os.getenv("REVISION", "").strip()
+    revision = runtime["revision"] or ""
-    api_key = os.getenv("API_KEY", "").strip()
+    api_key = runtime["api_key"] or ""
     cmd = [
         sys.executable,
         "-m",

config.json

@@ -1,5 +1,20 @@
 {
+  "services": {
+    "api": {
+      "host": "0.0.0.0",
+      "port": 8002
+    },
+    "openai": {
+      "host": "0.0.0.0",
+      "port": 8001
+    }
+  },
+  "api_key": "",
+  "tensor_parallel_size": 2,
+  "dtype": "bfloat16",
+  "revision": "",
   "default_model": "Qwen3.5-35B-A3B-GPTQ-Int4",
+  "selected_model": "Qwen3.5-35B-A3B-GPTQ-Int4",
   "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
     "ctx": "24576",
     "trust_remote": true,

docker-compose.yml

@@ -5,10 +5,9 @@ services:
       dockerfile: Dockerfile
     image: rocm-vllm-inference:latest
     container_name: rocm-vllm-inference
+    command: ["python", "-m", "app.start_api"]
     ports:
       - "8000:8000"
-    env_file:
-      - .env
     volumes:
       - /opt/model:/opt/model:ro
       - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro
@@ -34,10 +33,6 @@ services:
     command: ["python", "-m", "app.start_openai"]
     ports:
       - "8001:8001"
-    env_file:
-      - .env
-    environment:
-      OPENAI_PORT: 8001
     volumes:
       - /opt/model:/opt/model:ro
       - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro

requirements.txt

@@ -1,4 +1,3 @@
 fastapi==0.116.1
 uvicorn==0.35.0
 pydantic==2.11.7
-pydantic-settings==2.11.0