x

README.md

@@ -43,7 +43,7 @@
 - `models.default`：默认模型名
 - `models.selected`：当前生效模型名
 - `models.profiles`：模型配置集合
-- 每个模型必须包含：`local_path`，并建议补充 `ctx`、`max_num_seqs`、`max_tokens`
+- 每个模型必须包含：`local_path`，并建议补充 `ctx`、`max_num_seqs`、`max_tokens`、`dtype`、`quantization`
 
 启动时会按以下优先级选模型：
 

app/model_catalog.py

@@ -100,6 +100,8 @@ def resolve_model_profile(
         "selected_model": model_key,
         "model_name": _resolve_profile_model_path(profile, model_root, model_key),
         "served_model_name": profile.get("served_model_name", model_key),
+        "dtype": _to_str(profile.get("dtype")),
+        "quantization": _to_str(profile.get("quantization")),
         "max_model_len": _to_int(profile.get("ctx"), 8192),
         "max_num_seqs": _to_int(profile.get("max_num_seqs"), 64),
         "max_tokens": _to_int(profile.get("max_tokens"), 4096),

app/start_openai.py

@@ -19,7 +19,8 @@ def build_command() -> list[str]:
     host = str(runtime["openai_host"])
     port = str(runtime["openai_port"])
     api_key = runtime["api_key"] or ""
-    dtype = str(runtime["dtype"])
+    dtype = str(updates["dtype"] or runtime["dtype"])
+    quantization = str(updates["quantization"] or "").strip()
     revision = runtime["revision"] or ""
     cmd = [
         sys.executable,
@@ -52,6 +53,8 @@ def build_command() -> list[str]:
         cmd.append("--enable-auto-tool-choice")
     if updates["tool_call_parser"]:
         cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
+    if quantization:
+        cmd.extend(["--quantization", quantization])
     if revision:
         cmd.extend(["--revision", revision])
     if api_key:

config.json

@@ -64,6 +64,8 @@
       },
       "Qwen3.5-35B-A3B-GPTQ-Int4": {
         "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4",
+        "dtype": "float16",
+        "quantization": "gptq_marlin",
         "ctx": "65536",
         "trust_remote": true,
         "valid_tp": [1, 2],