diff --git a/config.json b/config.json index 6f82e80..b31181d 100644 --- a/config.json +++ b/config.json @@ -51,12 +51,13 @@ }, "Qwen3.5-27B-FP8": { "local_path": "Qwen3.5-27B-FP8", + "dtype": "bfloat16", "ctx": "65536", "trust_remote": true, "valid_tp": [1, 2], - "max_num_seqs": "8", - "max_tokens": "32768", - "gpu_util": "0.98", + "max_num_seqs": "4", + "max_tokens": "8192", + "gpu_util": "0.94", "tool_call_parser": "qwen3_xml", "enable_auto_tool_choice": true, "served_model_name": "Qwen3.5-27B-FP8",