cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
SZCJW
2026-03-27 01:08:25 +08:00
parent b5076cb33a
commit 5fad417889
1 changed files with 41 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.json
+41 -37
View File
@@ -1,57 +1,61 @@
{
"default_model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"default_model": "Qwen3.5-35B-A3B-GPTQ-Int4",
"models": {
"meta-llama/Meta-Llama-3.1-8B-Instruct": {
"ctx": "65536",
"trust_remote": false,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
},
"openai/gpt-oss-20b": {
"gpt-oss-20b": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "8192"
"max_tokens": "8192",
"hf_model_id": "openai/gpt-oss-20b"
},
"RedHatAI/Qwen3-14B-FP8-dynamic": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768"
},
"cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
"Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
"ctx": "24576",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
"max_tokens": "32768",
"hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit"
},
"cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
"ctx": "20480",
"DeepSeek-R1-Distill-Qwen-14B": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "16384",
"enforce_eager": false,
"env": {"VLLM_USE_TRITON_AWQ": "1"}
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768",
"hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"RedHatAI/gemma-3-27b-it-FP8-dynamic": {
"ctx": "29000",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "29000",
"gpu_util": "0.94"
},
"RedHatAI/gemma-3-12b-it-FP8-dynamic": {
"ctx": "9900",
"DeepSeek-R1-Distill-Qwen-32B-AWQ": {
"ctx": "24576",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "9900"
"max_tokens": "32768",
"hf_model_id": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B-AWQ"
},
"GLM-4.7-Flash-AWQ": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"hf_model_id": "THUDM/GLM-4.7-Flash-AWQ"
},
"Qwen3.5-35B-A3B-GPTQ-Int4": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
},
"Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit": {
"ctx": "24576",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768",
"hf_model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct-AWQ-4bit"
}
}
}