x

README.md

@@ -104,5 +104,6 @@ curl -X POST "http://localhost:8001/v1/chat/completions" \
 ## 常见故障排查
 
 - 报错 `model type ... Transformers does not recognize this architecture` 时，说明当前模型与镜像内依赖不兼容，建议更换模型或升级镜像版本。
+- 报错 `model config (gptq) does not match quantization argument (gptq_marlin)` 时，将该模型配置改为 `dtype=float16` 且 `quantization=gptq`。
 - 若模型目录存在但仍加载失败，检查挂载路径是否为 `/opt/model:/opt/model:ro`，并确认容器内可见模型文件。
 - 如果看到 `No services to build`，说明未触发重建；需要先执行 `docker compose build --no-cache` 再 `up`。

config.json

@@ -65,7 +65,7 @@
       "Qwen3.5-35B-A3B-GPTQ-Int4": {
         "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4",
         "dtype": "float16",
-        "quantization": "gptq_marlin",
+        "quantization": "gptq",
         "ctx": "65536",
         "trust_remote": true,
         "valid_tp": [1, 2],
@@ -76,6 +76,20 @@
         "enable_auto_tool_choice": true,
         "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",
         "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
+      },
+      "Qwen3.5-35B-A3B-FP8": {
+        "local_path": "Qwen3.5-35B-A3B-FP8",
+        "dtype": "bfloat16",
+        "ctx": "65536",
+        "trust_remote": true,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "32768",
+        "gpu_util": "0.98",
+        "tool_call_parser": "qwen3_xml",
+        "enable_auto_tool_choice": true,
+        "served_model_name": "Qwen3.5-35B-A3B-FP8",
+        "hf_model_id": "Qwen/Qwen3.5-35B-A3B-FP8"
       }
     }
   }