diff --git a/config.json b/config.json index 718eddc..43efba6 100644 --- a/config.json +++ b/config.json @@ -99,11 +99,10 @@ "dtype": "auto", "ctx": "131072", "trust_remote": true, - "valid_tp": [2], + "valid_tp": [1, 2], "max_num_seqs": "8", "max_tokens": "65536", "max_num_batched_tokens": 16384, - "kv_cache_dtype": "fp8", "gpu_util": "0.94", "enforce_eager": false, "enable_prefix_caching": true,