diff --git a/README.md b/README.md index f164fa6..c8160eb 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ - `models.default`:默认模型名 - `models.selected`:当前生效模型名 - `models.profiles`:模型配置集合 -- 每个模型必须包含:`local_path`,并建议补充 `ctx`、`max_num_seqs`、`max_tokens` +- 每个模型必须包含:`local_path`,并建议补充 `ctx`、`max_num_seqs`、`max_tokens`、`dtype`、`quantization` 启动时会按以下优先级选模型: diff --git a/app/model_catalog.py b/app/model_catalog.py index f5c4e15..75fc3e8 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -100,6 +100,8 @@ def resolve_model_profile( "selected_model": model_key, "model_name": _resolve_profile_model_path(profile, model_root, model_key), "served_model_name": profile.get("served_model_name", model_key), + "dtype": _to_str(profile.get("dtype")), + "quantization": _to_str(profile.get("quantization")), "max_model_len": _to_int(profile.get("ctx"), 8192), "max_num_seqs": _to_int(profile.get("max_num_seqs"), 64), "max_tokens": _to_int(profile.get("max_tokens"), 4096), diff --git a/app/start_openai.py b/app/start_openai.py index b6b13d5..6df4540 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -19,7 +19,8 @@ def build_command() -> list[str]: host = str(runtime["openai_host"]) port = str(runtime["openai_port"]) api_key = runtime["api_key"] or "" - dtype = str(runtime["dtype"]) + dtype = str(updates["dtype"] or runtime["dtype"]) + quantization = str(updates["quantization"] or "").strip() revision = runtime["revision"] or "" cmd = [ sys.executable, @@ -52,6 +53,8 @@ def build_command() -> list[str]: cmd.append("--enable-auto-tool-choice") if updates["tool_call_parser"]: cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])]) + if quantization: + cmd.extend(["--quantization", quantization]) if revision: cmd.extend(["--revision", revision]) if api_key: diff --git a/config.json b/config.json index cf80d36..c0462c0 100644 --- a/config.json +++ b/config.json @@ -64,6 +64,8 @@ }, "Qwen3.5-35B-A3B-GPTQ-Int4": { "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4", + "dtype": "float16", + "quantization": "gptq_marlin", "ctx": "65536", "trust_remote": true, "valid_tp": [1, 2],