From 645358862a0f105b6ace516521277544d25f40b1 Mon Sep 17 00:00:00 2001 From: SZCJW <792430652@qq.com> Date: Sun, 29 Mar 2026 05:31:19 +0800 Subject: [PATCH] x --- .env.example | 19 ------------- Dockerfile | 11 +------- README.md | 43 +++++++++++------------------ app/config.py | 64 ++++++++++++++++++++++++++------------------ app/model_catalog.py | 33 +++++++++++++++++++---- app/start_api.py | 23 ++++++++++++++++ app/start_openai.py | 24 ++++++++--------- config.json | 15 +++++++++++ docker-compose.yml | 7 +---- requirements.txt | 1 - 10 files changed, 134 insertions(+), 106 deletions(-) delete mode 100644 .env.example create mode 100644 app/start_api.py diff --git a/.env.example b/.env.example deleted file mode 100644 index 20bd71c..0000000 --- a/.env.example +++ /dev/null @@ -1,19 +0,0 @@ -MODEL_CONFIG_FILE=config.json -MODEL_KEY=Qwen3.5-35B-A3B-GPTQ-Int4 -MODEL_NAME= -HOST=0.0.0.0 -PORT=8000 -OPENAI_HOST=0.0.0.0 -OPENAI_PORT=8001 -TENSOR_PARALLEL_SIZE=2 -GPU_MEMORY_UTILIZATION=0.92 -MAX_MODEL_LEN=8192 -MAX_NUM_SEQS=64 -MAX_TOKENS=4096 -DTYPE=bfloat16 -ENFORCE_EAGER=false -TRUST_REMOTE_CODE=false -TOOL_CALL_PARSER= -ENABLE_AUTO_TOOL_CHOICE=false -REVISION= -API_KEY= diff --git a/Dockerfile b/Dockerfile index 2ddb35f..8983ab3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,16 +8,7 @@ RUN pip install --no-cache-dir -r /workspace/requirements.txt COPY app /workspace/app ENV PYTHONUNBUFFERED=1 -ENV HOST=0.0.0.0 -ENV PORT=8000 -ENV TENSOR_PARALLEL_SIZE=2 -ENV GPU_MEMORY_UTILIZATION=0.92 -ENV MAX_MODEL_LEN=8192 -ENV MAX_NUM_SEQS=64 -ENV DTYPE=bfloat16 -ENV ENFORCE_EAGER=false -ENV TRUST_REMOTE_CODE=false EXPOSE 8000 8001 -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["python", "-m", "app.start_api"] diff --git a/README.md b/README.md index 60c6a53..45a6593 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,10 @@ │ ├── engine.py │ ├── main.py │ ├── model_catalog.py +│ ├── start_api.py +│ ├── start_openai.py │ └── schemas.py ├── .dockerignore -├── .env.example ├── config.json ├── docker-compose.yml ├── Dockerfile @@ -53,20 +54,14 @@ ## 配置项 -可通过 `.env` 配置，建议先复制 `.env.example`： +项目只读取一个配置文件：`config.json`。 -- `MODEL_CONFIG_FILE`：模型配置文件路径，默认 `config.json` -- `MODEL_KEY`：要启动的模型键，留空时使用 `config.json` 中 `default_model` -- `MODEL_NAME`：兜底模型名（仅在不使用配置文件时生效） -- `OPENAI_HOST`：OpenAI 协议服务监听地址，默认 `0.0.0.0` -- `OPENAI_PORT`：OpenAI 协议服务监听端口，默认 `8001` -- `TENSOR_PARALLEL_SIZE`：张量并行数，双卡建议为 `2` -- `GPU_MEMORY_UTILIZATION`：单卡显存利用率，建议 `0.90~0.95` -- `MAX_MODEL_LEN`：模型最大上下文长度 -- `MAX_NUM_SEQS`：并发序列数量 -- `MAX_TOKENS`：单请求最大生成长度 -- `DTYPE`：精度类型，默认 `bfloat16` -- `API_KEY`：可选接口访问密钥 +- `services.api.host` / `services.api.port`：自定义推理服务监听地址与端口（默认 `0.0.0.0:8000`） +- `services.openai.host` / `services.openai.port`：OpenAI 协议服务监听地址与端口（默认 `0.0.0.0:8001`） +- `api_key`：接口访问密钥（同时用于 8000 与 8001） +- `tensor_parallel_size`：张量并行数，双卡建议 `2` +- `dtype`：推理精度，默认 `bfloat16` +- `selected_model`：当前生效模型，留空时回退到 `default_model` ## config.json 说明 @@ -78,7 +73,7 @@ 启动时会按以下优先级选模型： -1. `.env` 中 `MODEL_KEY` +1. `config.json` 中 `selected_model` 2. `config.json` 中 `default_model` 模型被选中后，会自动覆盖运行参数，包括： @@ -93,27 +88,21 @@ ## 部署步骤 -1. 准备环境变量： +1. 修改 `config.json` 中的 `selected_model` 与服务参数。 -```bash -cp .env.example .env -``` - -2. 修改 `.env` 中的 `MODEL_KEY`，选择要启动的模型。 - -3. 构建并启动容器： +2. 构建并启动容器： ```bash docker compose up -d --build ``` -4. 验证自定义推理服务： +3. 验证自定义推理服务： ```bash curl http://localhost:8000/health ``` -5. 验证 OpenAI 协议服务： +4. 验证 OpenAI 协议服务： ```bash curl http://localhost:8001/v1/models @@ -132,14 +121,14 @@ curl -X POST "http://localhost:8000/v1/generate" \ ```bash curl -X POST "http://localhost:8001/v1/chat/completions" \ -H "Content-Type: application/json" \ - -H "Authorization: Bearer ${API_KEY}" \ + -H "Authorization: Bearer " \ -d "{\"model\":\"Qwen3.5-35B-A3B-GPTQ-Int4\",\"messages\":[{\"role\":\"user\",\"content\":\"你好，介绍一下你自己\"}],\"temperature\":0.7}" ``` ## OpenClaw 调用说明 - Base URL 使用 `http://<服务器IP>:8001/v1` -- API Key 使用 `.env` 中 `API_KEY` 对应值 +- API Key 使用 `config.json` 中 `api_key` - 模型名使用 `config.json` 中 `served_model_name` - 若使用工具调用，`config.json` 中应配置 `tool_call_parser` 与 `enable_auto_tool_choice` diff --git a/app/config.py b/app/config.py index 5b9bb9d..9ddf929 100644 --- a/app/config.py +++ b/app/config.py @@ -2,44 +2,56 @@ from functools import lru_cache import os from typing import Optional -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel -from app.model_catalog import resolve_model_profile +from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings -class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") - - config_file: str = Field(default="config.json", alias="MODEL_CONFIG_FILE") - model_key: Optional[str] = Field(default=None, alias="MODEL_KEY") +class Settings(BaseModel): + config_file: str = "config.json" + model_key: Optional[str] = None selected_model: Optional[str] = None - model_name: str = Field(default="", alias="MODEL_NAME") + model_name: str = "" served_model_name: Optional[str] = None - host: str = Field(default="0.0.0.0", alias="HOST") - port: int = Field(default=8000, alias="PORT") - max_model_len: int = Field(default=8192, alias="MAX_MODEL_LEN") - gpu_memory_utilization: float = Field(default=0.92, alias="GPU_MEMORY_UTILIZATION") - tensor_parallel_size: int = Field(default=2, alias="TENSOR_PARALLEL_SIZE") - max_num_seqs: int = Field(default=64, alias="MAX_NUM_SEQS") - max_tokens: int = Field(default=4096, alias="MAX_TOKENS") - dtype: str = Field(default="bfloat16", alias="DTYPE") - enforce_eager: bool = Field(default=False, alias="ENFORCE_EAGER") - trust_remote_code: bool = Field(default=False, alias="TRUST_REMOTE_CODE") - tool_call_parser: Optional[str] = Field(default=None, alias="TOOL_CALL_PARSER") - enable_auto_tool_choice: bool = Field(default=False, alias="ENABLE_AUTO_TOOL_CHOICE") - revision: Optional[str] = Field(default=None, alias="REVISION") - api_key: Optional[str] = Field(default=None, alias="API_KEY") + host: str = "0.0.0.0" + port: int = 8000 + openai_host: str = "0.0.0.0" + openai_port: int = 8001 + max_model_len: int = 8192 + gpu_memory_utilization: float = 0.92 + tensor_parallel_size: int = 2 + max_num_seqs: int = 64 + max_tokens: int = 4096 + dtype: str = "bfloat16" + enforce_eager: bool = False + trust_remote_code: bool = False + tool_call_parser: Optional[str] = None + enable_auto_tool_choice: bool = False + revision: Optional[str] = None + api_key: Optional[str] = None @lru_cache(maxsize=1) def get_settings() -> Settings: - settings = Settings() + catalog = load_catalog("config.json") + runtime = resolve_runtime_settings(catalog) + settings = Settings( + config_file="config.json", + model_key=runtime["model_key"], + host=runtime["host"], + port=runtime["port"], + openai_host=runtime["openai_host"], + openai_port=runtime["openai_port"], + api_key=runtime["api_key"], + tensor_parallel_size=runtime["tensor_parallel_size"], + dtype=runtime["dtype"], + revision=runtime["revision"], + ) _, updates, env_vars = resolve_model_profile( - catalog_path=settings.config_file, + content=catalog, requested_model=settings.model_key, requested_tp=settings.tensor_parallel_size, ) for key, value in env_vars.items(): os.environ[key] = value - return settings.model_copy(update=updates) + return settings.model_copy(update=updates | runtime) diff --git a/app/model_catalog.py b/app/model_catalog.py index 08d827f..72d1864 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -29,14 +29,37 @@ def _to_float(value: Any, default: float) -> float: return default -def resolve_model_profile( - catalog_path: str, requested_model: str | None, requested_tp: int -) -> tuple[str, dict[str, Any], dict[str, str]]: +def load_catalog(catalog_path: str = "config.json") -> dict[str, Any]: content = json.loads(Path(catalog_path).read_text(encoding="utf-8")) if not isinstance(content, dict): raise ValueError("config.json must be a JSON object") - default_model = content.get("default_model") - profiles = {k: v for k, v in content.items() if k != "default_model"} + return content + + +def resolve_runtime_settings(content: dict[str, Any]) -> dict[str, Any]: + services = content.get("services", {}) + api_service = dict(services.get("api", {})) + openai_service = dict(services.get("openai", {})) + models = dict(content.get("models", {})) + return { + "host": str(api_service.get("host", "0.0.0.0")), + "port": _to_int(api_service.get("port"), 8000), + "openai_host": str(openai_service.get("host", "0.0.0.0")), + "openai_port": _to_int(openai_service.get("port"), 8001), + "api_key": str(content.get("api_key", "")).strip() or None, + "tensor_parallel_size": _to_int(content.get("tensor_parallel_size"), 2), + "dtype": str(content.get("dtype", "bfloat16")), + "revision": str(content.get("revision", "")).strip() or None, + "model_key": str(models.get("selected", "")).strip() or None, + } + + +def resolve_model_profile( + content: dict[str, Any], requested_model: str | None, requested_tp: int +) -> tuple[str, dict[str, Any], dict[str, str]]: + models = dict(content.get("models", {})) + profiles = dict(models.get("profiles", {})) + default_model = models.get("default") model_key = requested_model or default_model if not model_key or model_key not in profiles: raise ValueError(f"model profile '{model_key}' not found in config.json") diff --git a/app/start_api.py b/app/start_api.py new file mode 100644 index 0000000..400e4c0 --- /dev/null +++ b/app/start_api.py @@ -0,0 +1,23 @@ +import subprocess +import sys + +from app.config import get_settings + + +def main() -> None: + settings = get_settings() + command = [ + sys.executable, + "-m", + "uvicorn", + "app.main:app", + "--host", + settings.host, + "--port", + str(settings.port), + ] + raise SystemExit(subprocess.call(command)) + + +if __name__ == "__main__": + main() diff --git a/app/start_openai.py b/app/start_openai.py index 1ae7b39..9e24f9f 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -2,25 +2,25 @@ import os import subprocess import sys -from app.model_catalog import resolve_model_profile +from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings def build_command() -> list[str]: - config_file = os.getenv("MODEL_CONFIG_FILE", "config.json") - model_key = os.getenv("MODEL_KEY") - requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2")) + config_file = "config.json" + catalog = load_catalog(config_file) + runtime = resolve_runtime_settings(catalog) _, updates, env_vars = resolve_model_profile( - catalog_path=config_file, - requested_model=model_key, - requested_tp=requested_tp, + content=catalog, + requested_model=runtime["model_key"], + requested_tp=runtime["tensor_parallel_size"], ) for key, value in env_vars.items(): os.environ[key] = value - host = os.getenv("OPENAI_HOST", "0.0.0.0") - port = os.getenv("OPENAI_PORT", "8001") - dtype = os.getenv("DTYPE", "bfloat16") - revision = os.getenv("REVISION", "").strip() - api_key = os.getenv("API_KEY", "").strip() + host = str(runtime["openai_host"]) + port = str(runtime["openai_port"]) + dtype = str(runtime["dtype"]) + revision = runtime["revision"] or "" + api_key = runtime["api_key"] or "" cmd = [ sys.executable, "-m", diff --git a/config.json b/config.json index 4fd9bdf..2159ac5 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,20 @@ { + "services": { + "api": { + "host": "0.0.0.0", + "port": 8002 + }, + "openai": { + "host": "0.0.0.0", + "port": 8001 + } + }, + "api_key": "", + "tensor_parallel_size": 2, + "dtype": "bfloat16", + "revision": "", "default_model": "Qwen3.5-35B-A3B-GPTQ-Int4", + "selected_model": "Qwen3.5-35B-A3B-GPTQ-Int4", "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { "ctx": "24576", "trust_remote": true, diff --git a/docker-compose.yml b/docker-compose.yml index 85a1d77..1e40c8d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,10 +5,9 @@ services: dockerfile: Dockerfile image: rocm-vllm-inference:latest container_name: rocm-vllm-inference + command: ["python", "-m", "app.start_api"] ports: - "8000:8000" - env_file: - - .env volumes: - /opt/model:/opt/model:ro - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro @@ -34,10 +33,6 @@ services: command: ["python", "-m", "app.start_openai"] ports: - "8001:8001" - env_file: - - .env - environment: - OPENAI_PORT: 8001 volumes: - /opt/model:/opt/model:ro - d:/project/rocm_vllm_nightly/rocm_vllm_nightly/config.json:/workspace/config.json:ro diff --git a/requirements.txt b/requirements.txt index 49a035e..3ae98d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ fastapi==0.116.1 uvicorn==0.35.0 pydantic==2.11.7 -pydantic-settings==2.11.0