diff --git a/config.json b/config.json index 4e88e99..b7fee89 100644 --- a/config.json +++ b/config.json @@ -101,7 +101,7 @@ "trust_remote": true, "valid_tp": [2], "max_num_seqs": "12", - "max_tokens": "16384", + "max_tokens": "65536", "gpu_util": "0.94", "enforce_eager": false, "enable_prefix_caching": true,