diff --git a/config.json b/config.json index d52fe3c..746c547 100644 --- a/config.json +++ b/config.json @@ -1,57 +1,61 @@ { - "default_model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "default_model": "Qwen3.5-35B-A3B-GPTQ-Int4", "models": { - "meta-llama/Meta-Llama-3.1-8B-Instruct": { - "ctx": "65536", - "trust_remote": false, - "valid_tp": [1, 2], - "max_num_seqs": "64", - "max_tokens": "32768" - }, - "openai/gpt-oss-20b": { + "gpt-oss-20b": { "ctx": "32768", "trust_remote": true, "valid_tp": [1, 2], "max_num_seqs": "64", - "max_tokens": "8192" + "max_tokens": "8192", + "hf_model_id": "openai/gpt-oss-20b" }, - "RedHatAI/Qwen3-14B-FP8-dynamic": { - "ctx": "32768", - "trust_remote": true, - "valid_tp": [1], - "max_num_seqs": "64", - "max_tokens": "32768" - }, - "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": { + "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": { "ctx": "24576", "trust_remote": true, "valid_tp": [1, 2], "max_num_seqs": "64", - "max_tokens": "32768" + "max_tokens": "32768", + "hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit" }, - "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { - "ctx": "20480", + "DeepSeek-R1-Distill-Qwen-14B": { + "ctx": "32768", "trust_remote": true, - "valid_tp": [2], - "max_num_seqs": "32", - "max_tokens": "16384", - "enforce_eager": false, - "env": {"VLLM_USE_TRITON_AWQ": "1"} + "valid_tp": [1], + "max_num_seqs": "64", + "max_tokens": "32768", + "hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B" }, - "RedHatAI/gemma-3-27b-it-FP8-dynamic": { - "ctx": "29000", - "trust_remote": true, - "valid_tp": [2], - "max_num_seqs": "32", - "max_tokens": "29000", - "gpu_util": "0.94" - }, - "RedHatAI/gemma-3-12b-it-FP8-dynamic": { - "ctx": "9900", + "DeepSeek-R1-Distill-Qwen-32B-AWQ": { + "ctx": "24576", "trust_remote": true, "valid_tp": [1, 2], "max_num_seqs": "64", - "max_tokens": "9900" + "max_tokens": "32768", + "hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B-AWQ" + }, + "GLM-4.7-Flash-AWQ": { + "ctx": "32768", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + "hf_model_id": "THUDM/GLM-4.7-Flash-AWQ" + }, + "Qwen3.5-35B-A3B-GPTQ-Int4": { + "ctx": "32768", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4" + }, + "Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit": { + "ctx": "24576", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + "hf_model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit" } } } \ No newline at end of file