1404 lines
37 KiB
JSON
1404 lines
37 KiB
JSON
{
|
|
"runs": [
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 186.47
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 59.68
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 129.43
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 61.28
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 57.73
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 25.29
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 14118.91
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 88.1
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 89.11
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 22.0
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1650.9257882903235
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 74.14
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 28.0
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 75.4
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 28.6
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1795.5908237208996
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 120.83
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 45.17
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1325.705122782343
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1878.7081268950026
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "AWQ",
|
|
"params_b": 80.0,
|
|
"name_params_b": 80.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 680.226281561525
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1360.3779166371198
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 12436.56
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 84.04
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1095.31715306375
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 222.29
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 54.74
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 135.44
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 47.24
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 58.57
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 21.86
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 52928.91
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 61.4
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 984.5314161719857
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 108.91
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 47.9
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1019.986034633913
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 85.92
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 42.81
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "AWQ",
|
|
"params_b": 80.0,
|
|
"name_params_b": 80.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 72209.01
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "AWQ",
|
|
"params_b": 80.0,
|
|
"name_params_b": 80.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 71.26
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 104.59
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 38.86
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "AWQ",
|
|
"params_b": 80.0,
|
|
"name_params_b": 80.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 519.96
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "AWQ",
|
|
"params_b": 80.0,
|
|
"name_params_b": 80.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 44.04
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 0.0
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 0.0
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 4078.97
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 78.49
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 80.34
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 37.1
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "default",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 859.7532189051877
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 93.13
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 47.81
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "default",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 2736.811464734795
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 123.92
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 73.89
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 118.15
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 54.56
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 86.17
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 28.21
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 14305.88
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 88.19
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 232.41
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 39.99
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1486.353457790027
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 97.06
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 32.9
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1890.096088342886
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 125.43
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 51.71
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1299.397329054036
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1575.9035072220947
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1429.9541207853842
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 3686.81
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 75.05
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1221.9380837920544
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 37309.58
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 113.18
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 235.33
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 55.13
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 80.83
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 19.52
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 0.0
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 0.0
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "FP8",
|
|
"params_b": 27.0,
|
|
"name_params_b": 27.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 621.3330493592667
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 104.14
|
|
},
|
|
{
|
|
"model": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 14.0,
|
|
"name_params_b": 14.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 41.41
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1024.3570195753498
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 81.58
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 37.28
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 127.38
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 45.15
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 1170.97
|
|
},
|
|
{
|
|
"model": "openai/gpt-oss-20b",
|
|
"model_clean": "openai/gpt-oss-20b",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 20.0,
|
|
"name_params_b": 20.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 73.13
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 1.0",
|
|
"tps_mean": 74.71
|
|
},
|
|
{
|
|
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "BF16",
|
|
"params_b": 8.0,
|
|
"name_params_b": 8.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 1.0",
|
|
"tps_mean": 30.85
|
|
},
|
|
{
|
|
"model": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic",
|
|
"env": "TP1",
|
|
"variant": "rocm",
|
|
"gpu_config": "single",
|
|
"quant": "FP8",
|
|
"params_b": 12.0,
|
|
"name_params_b": 12.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "Throughput",
|
|
"tps_mean": 1132.6153536237514
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TTFT @ QPS 4.0",
|
|
"tps_mean": 194.04
|
|
},
|
|
{
|
|
"model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
|
|
"env": "TP2",
|
|
"variant": "rocm",
|
|
"gpu_config": "dual",
|
|
"quant": "GPTQ",
|
|
"params_b": 30.0,
|
|
"name_params_b": 30.0,
|
|
"backend": "vLLM",
|
|
"error": false,
|
|
"test": "TPOT @ QPS 4.0",
|
|
"tps_mean": 58.93
|
|
}
|
|
]
|
|
} |