diff --git a/config.json b/config.json index 633a462..b05bf04 100644 --- a/config.json +++ b/config.json @@ -20,7 +20,7 @@ "revision": "", "models": { "default": "Qwen3.5-35B-A3B-GPTQ-Int4", - "selected": "Qwen3.6-27B-FP8", + "selected": "Qwen3.6-35B-A3B-FP8", "profiles": { "Qwen3.6-35B-A3B-FP8": { "local_path": "Qwen3.6-35B-A3B-FP8", @@ -28,14 +28,14 @@ "quantization": "fp8", "ctx": "262144", "max_tokens": "65536", - "max_num_batched_tokens": 16384, + "max_num_batched_tokens": 32768, "trust_remote": true, "enforce_eager": false, "valid_tp": [ 2 ], - "max_num_seqs": "12", - "gpu_util": "0.90", + "max_num_seqs": "256", + "gpu_util": "0.92", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "enable_auto_tool_choice": true, @@ -69,18 +69,19 @@ "local_path": "Qwen3.6-27B-FP8", "dtype": "auto", "quantization": "fp8", - "ctx": "32768", - "max_tokens": "16384", - "max_num_batched_tokens": 8192, + "ctx": "262144", + "max_tokens": "65536", + "max_num_batched_tokens": 32768, "trust_remote": true, "enforce_eager": false, "valid_tp": [ + 1, 2 ], - "max_num_seqs": 16, - "gpu_util": "0.80", + "max_num_seqs": 256, + "gpu_util": "0.92", "kv_cache_dtype": "fp8", - "enable_prefix_caching": false, + "enable_prefix_caching": true, "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "enable_auto_tool_choice": true,