diff --git a/config.json b/config.json index 692a6b6..ca9c0ca 100644 --- a/config.json +++ b/config.json @@ -71,14 +71,14 @@ "quantization": "fp8", "ctx": "262144", "max_tokens": "65536", - "max_num_batched_tokens": 16384, + "max_num_batched_tokens": 32768, "trust_remote": true, "enforce_eager": false, "valid_tp": [ 1, 2 ], - "max_num_seqs": 32, + "max_num_seqs": 256, "gpu_util": "0.92", "kv_cache_dtype": "fp8", "enable_prefix_caching": true,