x

2026-03-29 17:32:12 +08:00
parent a339629039
commit 7ff076530a
2 changed files with 16 additions and 1 deletions
@@ -65,7 +65,7 @@
      "Qwen3.5-35B-A3B-GPTQ-Int4": {
        "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4",
        "dtype": "float16",
-        "quantization": "gptq_marlin",
+        "quantization": "gptq",
        "ctx": "65536",
        "trust_remote": true,
        "valid_tp": [1, 2],
@@ -76,6 +76,20 @@
        "enable_auto_tool_choice": true,
        "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",
        "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
+      },
+      "Qwen3.5-35B-A3B-FP8": {
+        "local_path": "Qwen3.5-35B-A3B-FP8",
+        "dtype": "bfloat16",
+        "ctx": "65536",
+        "trust_remote": true,
+        "valid_tp": [1, 2],
+        "max_num_seqs": "64",
+        "max_tokens": "32768",
+        "gpu_util": "0.98",
+        "tool_call_parser": "qwen3_xml",
+        "enable_auto_tool_choice": true,
+        "served_model_name": "Qwen3.5-35B-A3B-FP8",
+        "hf_model_id": "Qwen/Qwen3.5-35B-A3B-FP8"
      }
    }
  }