x

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f88cf679990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-66e0ceca-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.88    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40286     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         214.42    \nPeak output token throughput (tok/s):    460.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          418.59    \n---------------Time to First Token----------------\nMean TTFT (ms):                          77.11     \nMedian TTFT (ms):                        52.45     \nP99 TTFT (ms):                           199.22    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          15.76     \nMedian TPOT (ms):                        15.43     \nP99 TPOT (ms):                           21.30     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.60     \nMedian ITL (ms):                         14.96     \nP99 ITL (ms):                            17.95     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f719e189990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-702c15c4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.21    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155632    \nRequest throughput (req/s):              3.79      \nOutput token throughput (tok/s):         818.19    \nPeak output token throughput (tok/s):    1520.00   \nPeak concurrent requests:                37.00     \nTotal Token throughput (tok/s):          1589.40   \n---------------Time to First Token----------------\nMean TTFT (ms):                          76.15     \nMedian TTFT (ms):                        45.58     \nP99 TTFT (ms):                           247.34    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          19.54     \nMedian TPOT (ms):                        18.83     \nP99 TPOT (ms):                           35.16     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           19.12     \nMedian ITL (ms):                         16.70     \nP99 ITL (ms):                            114.66    \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 248.4086626805365,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 4.0256245060424485,
+    "tokens_per_second": 2984.3323175624723
+}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc3c8c05990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b80b1bb5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.14    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39211     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.94    \nPeak output token throughput (tok/s):    560.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          421.26    \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.12     \nMedian TTFT (ms):                        41.56     \nP99 TTFT (ms):                           82.51     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.89      \nMedian TPOT (ms):                        8.73      \nP99 TPOT (ms):                           13.87     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.70      \nMedian ITL (ms):                         8.44      \nP99 ITL (ms):                            11.78     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ff610411990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fb44348e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.80    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153333    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         820.85    \nPeak output token throughput (tok/s):    1449.00   \nPeak concurrent requests:                31.00     \nTotal Token throughput (tok/s):          1606.16   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.68     \nMedian TTFT (ms):                        36.77     \nP99 TTFT (ms):                           89.21     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.46     \nMedian TPOT (ms):                        14.20     \nP99 TPOT (ms):                           19.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         13.63     \nP99 ITL (ms):                            44.44     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 199.3360565919429,
+    "num_requests": 1000,
+    "total_num_tokens": 741334,
+    "requests_per_second": 5.016653871341908,
+    "tokens_per_second": 3719.016081057382
+}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fe9a3b39990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4225f4cd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.19    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38830     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         207.43    \nPeak output token throughput (tok/s):    490.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          409.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          45.84     \nMedian TTFT (ms):                        37.87     \nP99 TTFT (ms):                           101.24    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.35     \nMedian TPOT (ms):                        14.27     \nP99 TPOT (ms):                           16.03     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         14.04     \nP99 ITL (ms):                            16.81     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f45831f1990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ca67e7d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.55    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152171    \nRequest throughput (req/s):              3.78      \nOutput token throughput (tok/s):         798.60    \nPeak output token throughput (tok/s):    1492.00   \nPeak concurrent requests:                35.00     \nTotal Token throughput (tok/s):          1563.82   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.19     \nMedian TTFT (ms):                        37.54     \nP99 TTFT (ms):                           101.54    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          16.05     \nMedian TPOT (ms):                        15.69     \nP99 TPOT (ms):                           20.99     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.95     \nMedian ITL (ms):                         15.02     \nP99 ITL (ms):                            41.81     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 195.21851312741637,
+    "num_requests": 1000,
+    "total_num_tokens": 736330,
+    "requests_per_second": 5.122464995660089,
+    "tokens_per_second": 3771.8246502543934
+}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc0cf5b5990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-85d6e157-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.05    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38779     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         211.85    \nPeak output token throughput (tok/s):    676.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          36.19     \nMedian TTFT (ms):                        30.20     \nP99 TTFT (ms):                           83.37     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.40      \nMedian TPOT (ms):                        7.32      \nP99 TPOT (ms):                           10.60     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.34      \nMedian ITL (ms):                         7.07      \nP99 ITL (ms):                            10.69     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -0,0 +1,4 @@
+{
+  "success": true,
+  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6542301990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-584a872e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  185.93    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151690    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         815.86    \nPeak output token throughput (tok/s):    1630.00   \nPeak concurrent requests:                28.00     \nTotal Token throughput (tok/s):          1598.63   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.36     \nMedian TTFT (ms):                        31.91     \nP99 TTFT (ms):                           78.67     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.67     \nMedian TPOT (ms):                        12.82     \nP99 TPOT (ms):                           16.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.51     \nMedian ITL (ms):                         12.62     \nP99 ITL (ms):                            31.79     \n==================================================\n"
+}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_throughput.json

@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 156.32366928458214,
+    "num_requests": 1000,
+    "total_num_tokens": 738792,
+    "requests_per_second": 6.396983928131399,
+    "tokens_per_second": 4726.040550232053
+}