diff --git a/.env.example b/.env.example index 0aabe69..839cc7b 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,17 @@ -MODEL_NAME=Qwen/Qwen3-14B +MODEL_CONFIG_FILE=config.json +MODEL_KEY=Qwen3.5-35B-A3B-GPTQ-Int4 +MODEL_NAME= HOST=0.0.0.0 PORT=8000 TENSOR_PARALLEL_SIZE=2 GPU_MEMORY_UTILIZATION=0.92 MAX_MODEL_LEN=8192 MAX_NUM_SEQS=64 +MAX_TOKENS=4096 DTYPE=bfloat16 ENFORCE_EAGER=false TRUST_REMOTE_CODE=false +TOOL_CALL_PARSER= +ENABLE_AUTO_TOOL_CHOICE=false REVISION= API_KEY= diff --git a/README.md b/README.md index 146e72f..3e792dd 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,11 @@ │ ├── config.py │ ├── engine.py │ ├── main.py +│ ├── model_catalog.py │ └── schemas.py ├── .dockerignore ├── .env.example +├── config.json ├── docker-compose.yml ├── Dockerfile └── requirements.txt @@ -52,14 +54,40 @@ 可通过 `.env` 配置,建议先复制 `.env.example`: -- `MODEL_NAME`:模型名称或本地路径 +- `MODEL_CONFIG_FILE`:模型配置文件路径,默认 `config.json` +- `MODEL_KEY`:要启动的模型键,留空时使用 `config.json` 中 `default_model` +- `MODEL_NAME`:兜底模型名(仅在不使用配置文件时生效) - `TENSOR_PARALLEL_SIZE`:张量并行数,双卡建议为 `2` - `GPU_MEMORY_UTILIZATION`:单卡显存利用率,建议 `0.90~0.95` - `MAX_MODEL_LEN`:模型最大上下文长度 - `MAX_NUM_SEQS`:并发序列数量 +- `MAX_TOKENS`:单请求最大生成长度 - `DTYPE`:精度类型,默认 `bfloat16` - `API_KEY`:可选接口访问密钥 +## config.json 说明 + +`config.json` 采用以下结构: + +- 顶层 `default_model`:默认模型名 +- 顶层其他键:每个模型的配置档位 +- 每个模型至少建议包含:`hf_model_id`、`ctx`、`max_num_seqs`、`max_tokens`、`gpu_util`、`valid_tp` + +启动时会按以下优先级选模型: + +1. `.env` 中 `MODEL_KEY` +2. `config.json` 中 `default_model` + +模型被选中后,会自动覆盖运行参数,包括: + +- `model_name` ← `hf_model_id` +- `max_model_len` ← `ctx` +- `max_num_seqs` ← `max_num_seqs` +- `max_tokens` ← `max_tokens` +- `gpu_memory_utilization` ← `gpu_util` +- `trust_remote_code` ← `trust_remote` +- `enforce_eager` ← `enforce_eager` + ## 部署步骤 1. 准备环境变量: @@ -68,7 +96,7 @@ cp .env.example .env ``` -2. 修改 `.env` 中的 `MODEL_NAME` 与资源参数。 +2. 修改 `.env` 中的 `MODEL_KEY`,选择要启动的模型。 3. 构建并启动容器: diff --git a/app/engine.py b/app/engine.py index 2916320..416f438 100644 --- a/app/engine.py +++ b/app/engine.py @@ -39,7 +39,7 @@ class InferenceEngine: return GenerateResponse( text=completion, prompt=req.prompt, - model=self.settings.model_name, + model=self.settings.served_model_name or self.settings.model_name, usage_prompt_tokens=usage_prompt, usage_completion_tokens=usage_completion, usage_total_tokens=usage_prompt + usage_completion, diff --git a/app/main.py b/app/main.py index 4fef1b3..74e86f1 100644 --- a/app/main.py +++ b/app/main.py @@ -33,11 +33,16 @@ app = FastAPI(title="ROCm vLLM Inference API", version="1.0.0", lifespan=lifespa @app.get("/health", response_model=HealthResponse) def health(settings: Settings = Depends(get_settings)) -> HealthResponse: - return HealthResponse(status="ok", model=settings.model_name) + return HealthResponse(status="ok", model=settings.served_model_name or settings.model_name) @app.post("/v1/generate", response_model=GenerateResponse, dependencies=[Depends(verify_api_key)]) -def generate(req: GenerateRequest) -> GenerateResponse: +def generate(req: GenerateRequest, settings: Settings = Depends(get_settings)) -> GenerateResponse: if engine is None: raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail="Engine not ready") + if req.max_tokens > settings.max_tokens: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"max_tokens must be <= {settings.max_tokens}", + ) return engine.generate(req)