cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b5076cb33ace0411b37a24fac377592e2f927184
amd-r9700-vllm-toolboxes/docs/results.json
T

1404 lines
37 KiB
JSON
Raw Normal View History

x
2026-03-26 20:50:00 +08:00
{
"runs": [
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 186.47
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 59.68
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 129.43
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 61.28
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 57.73
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 25.29
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 14118.91
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 88.1
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 89.11
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 22.0
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1650.9257882903235
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 74.14
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 28.0
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 75.4
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 28.6
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1795.5908237208996
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 120.83
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 45.17
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1325.705122782343
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1878.7081268950026
},
{
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "AWQ",
"params_b": 80.0,
"name_params_b": 80.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 680.226281561525
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1360.3779166371198
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 12436.56
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 84.04
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1095.31715306375
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 222.29
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 54.74
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 135.44
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 47.24
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 58.57
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 21.86
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 52928.91
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 61.4
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 984.5314161719857
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 108.91
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 47.9
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1019.986034633913
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 85.92
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 42.81
},
{
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "AWQ",
"params_b": 80.0,
"name_params_b": 80.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 72209.01
},
{
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "AWQ",
"params_b": 80.0,
"name_params_b": 80.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 71.26
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 104.59
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 38.86
},
{
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "AWQ",
"params_b": 80.0,
"name_params_b": 80.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 519.96
},
{
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "AWQ",
"params_b": 80.0,
"name_params_b": 80.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 44.04
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 0.0
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 0.0
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 4078.97
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 78.49
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 80.34
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 37.1
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "default",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 859.7532189051877
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 93.13
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 47.81
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "default",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 2736.811464734795
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 123.92
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 73.89
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 118.15
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 54.56
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 86.17
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 28.21
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 14305.88
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 88.19
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 232.41
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 39.99
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1486.353457790027
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 97.06
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 32.9
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1890.096088342886
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 125.43
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 51.71
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1299.397329054036
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1575.9035072220947
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1429.9541207853842
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 3686.81
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 75.05
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1221.9380837920544
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 37309.58
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 113.18
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 235.33
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 55.13
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 80.83
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 19.52
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 0.0
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 0.0
},
{
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "FP8",
"params_b": 27.0,
"name_params_b": 27.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 621.3330493592667
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 104.14
},
{
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 14.0,
"name_params_b": 14.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 41.41
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1024.3570195753498
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 81.58
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 37.28
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 127.38
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 45.15
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 1170.97
},
{
"model": "openai/gpt-oss-20b",
"model_clean": "openai/gpt-oss-20b",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 20.0,
"name_params_b": 20.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 73.13
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 1.0",
"tps_mean": 74.71
},
{
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "BF16",
"params_b": 8.0,
"name_params_b": 8.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 1.0",
"tps_mean": 30.85
},
{
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
"env": "TP1",
"variant": "rocm",
"gpu_config": "single",
"quant": "FP8",
"params_b": 12.0,
"name_params_b": 12.0,
"backend": "vLLM",
"error": false,
"test": "Throughput",
"tps_mean": 1132.6153536237514
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TTFT @ QPS 4.0",
"tps_mean": 194.04
},
{
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"env": "TP2",
"variant": "rocm",
"gpu_config": "dual",
"quant": "GPTQ",
"params_b": 30.0,
"name_params_b": 30.0,
"backend": "vLLM",
"error": false,
"test": "TPOT @ QPS 4.0",
"tps_mean": 58.93
}
]
}
Reference in New Issue Copy Permalink