init

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Llama-3.1-8B-Instruct-FP8-block_tp1_server.log

@@ -1,95 +0,0 @@
-WARNING 12-10 09:53:36 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
-(APIServer pid=4288) INFO 12-10 09:53:36 [api_server.py:1772] vLLM API server version 0.12.0
-(APIServer pid=4288) INFO 12-10 09:53:36 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'host': '127.0.0.1', 'model': 'RedHatAI/Llama-3.1-8B-Instruct-FP8-block', 'trust_remote_code': True, 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64}
-(APIServer pid=4288) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
-(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:637] Resolved architecture: LlamaForCausalLM
-(APIServer pid=4288) INFO 12-10 09:53:47 [model.py:1750] Using max model len 65536
-(APIServer pid=4288) INFO 12-10 09:53:47 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
-(APIServer pid=4288) Traceback (most recent call last):
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 409, in hf_raise_for_status
-(APIServer pid=4288)     response.raise_for_status()
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/requests/models.py", line 1026, in raise_for_status
-(APIServer pid=4288)     raise HTTPError(http_error_msg, response=self)
-(APIServer pid=4288) requests.exceptions.HTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block
-(APIServer pid=4288) 
-(APIServer pid=4288) The above exception was the direct cause of the following exception:
-(APIServer pid=4288) 
-(APIServer pid=4288) Traceback (most recent call last):
-(APIServer pid=4288)   File "/venv/main/bin/vllm", line 7, in <module>
-(APIServer pid=4288)     sys.exit(main())
-(APIServer pid=4288)              ^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
-(APIServer pid=4288)     args.dispatch_function(args)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
-(APIServer pid=4288)     uvloop.run(run_server(args))
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run
-(APIServer pid=4288)     return __asyncio.run(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run
-(APIServer pid=4288)     return runner.run(main)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run
-(APIServer pid=4288)     return self._loop.run_until_complete(task)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper
-(APIServer pid=4288)     return await main
-(APIServer pid=4288)            ^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server
-(APIServer pid=4288)     await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker
-(APIServer pid=4288)     async with build_async_engine_client(
-(APIServer pid=4288)                ^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=4288)     return await anext(self.gen)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client
-(APIServer pid=4288)     async with build_async_engine_client_from_engine_args(
-(APIServer pid=4288)                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__
-(APIServer pid=4288)     return await anext(self.gen)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args
-(APIServer pid=4288)     async_llm = AsyncLLM.from_vllm_config(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config
-(APIServer pid=4288)     return cls(
-(APIServer pid=4288)            ^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__
-(APIServer pid=4288)     tokenizer = init_tokenizer_from_config(self.model_config)
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config
-(APIServer pid=4288)     return get_tokenizer(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer
-(APIServer pid=4288)     tokenizer = TokenizerRegistry.get_tokenizer(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer
-(APIServer pid=4288)     return item.from_pretrained(*args, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained
-(APIServer pid=4288)     tokenizer = AutoTokenizer.from_pretrained(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained
-(APIServer pid=4288)     return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained
-(APIServer pid=4288)     return cls._from_pretrained(
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained
-(APIServer pid=4288)     tokenizer = cls._patch_mistral_regex(
-(APIServer pid=4288)                 ^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex
-(APIServer pid=4288)     if _is_local or is_base_mistral(pretrained_model_name_or_path):
-(APIServer pid=4288)                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral
-(APIServer pid=4288)     model = model_info(model_id)
-(APIServer pid=4288)             ^^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-(APIServer pid=4288)     return fn(*args, **kwargs)
-(APIServer pid=4288)            ^^^^^^^^^^^^^^^^^^^
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info
-(APIServer pid=4288)     hf_raise_for_status(r)
-(APIServer pid=4288)   File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status
-(APIServer pid=4288)     raise _format(HfHubHTTPError, str(e), response) from e
-(APIServer pid=4288) huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73dcacabf4c0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  191.24    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         210.71    \nPeak output token throughput (tok/s):    430.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          411.29    \n---------------Time to First Token----------------\nMean TTFT (ms):                          145.79    \nMedian TTFT (ms):                        110.27    \nP99 TTFT (ms):                           399.40    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.46     \nMedian TPOT (ms):                        22.67     \nP99 TPOT (ms):                           36.03     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.37     \nMedian ITL (ms):                         21.34     \nP99 ITL (ms):                            84.18     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cabb891f240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  215.08    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155647    \nRequest throughput (req/s):              3.35      \nOutput token throughput (tok/s):         723.67    \nPeak output token throughput (tok/s):    1208.00   \nPeak concurrent requests:                113.00    \nTotal Token throughput (tok/s):          1405.72   \n---------------Time to First Token----------------\nMean TTFT (ms):                          11329.86  \nMedian TTFT (ms):                        14049.92  \nP99 TTFT (ms):                           20450.77  \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          40.85     \nMedian TPOT (ms):                        39.82     \nP99 TPOT (ms):                           83.14     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           40.32     \nMedian ITL (ms):                         26.70     \nP99 ITL (ms):                            338.67    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 1446.3949257107452,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 0.6913741069083253,
-    "tokens_per_second": 512.5391321707765
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7efdf61eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-db5fc148-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.17    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40015     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         217.27    \nPeak output token throughput (tok/s):    541.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          425.55    \n---------------Time to First Token----------------\nMean TTFT (ms):                          58.17     \nMedian TTFT (ms):                        40.63     \nP99 TTFT (ms):                           140.17    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          9.73      \nMedian TPOT (ms):                        9.55      \nP99 TPOT (ms):                           15.94     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           9.55      \nMedian ITL (ms):                         9.45      \nP99 ITL (ms):                            14.11     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7aa491426fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c07c837c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  189.59    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153522    \nRequest throughput (req/s):              3.80      \nOutput token throughput (tok/s):         809.76    \nPeak output token throughput (tok/s):    1387.00   \nPeak concurrent requests:                40.00     \nTotal Token throughput (tok/s):          1583.51   \n---------------Time to First Token----------------\nMean TTFT (ms):                          83.62     \nMedian TTFT (ms):                        71.44     \nP99 TTFT (ms):                           196.63    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          22.34     \nMedian TPOT (ms):                        21.97     \nP99 TPOT (ms):                           35.34     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           21.92     \nMedian ITL (ms):                         19.31     \nP99 ITL (ms):                            107.58    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 306.69494965299964,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 3.2605688523121055,
-    "tokens_per_second": 2417.170549559942
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x78113ee3efc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bec359dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  191.18    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38760     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         202.74    \nPeak output token throughput (tok/s):    418.00    \nPeak concurrent requests:                11.00     \nTotal Token throughput (tok/s):          400.67    \n---------------Time to First Token----------------\nMean TTFT (ms):                          95.72     \nMedian TTFT (ms):                        55.76     \nP99 TTFT (ms):                           257.24    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.29     \nMedian TPOT (ms):                        23.04     \nP99 TPOT (ms):                           28.71     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.29     \nMedian ITL (ms):                         22.01     \nP99 ITL (ms):                            67.09     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff14b2efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-acbcf111-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  199.30    \nTotal input tokens:                      145810    \nTotal generated tokens:                  151171    \nRequest throughput (req/s):              3.61      \nOutput token throughput (tok/s):         758.52    \nPeak output token throughput (tok/s):    1326.00   \nPeak concurrent requests:                52.00     \nTotal Token throughput (tok/s):          1490.14   \n---------------Time to First Token----------------\nMean TTFT (ms):                          134.75    \nMedian TTFT (ms):                        130.98    \nP99 TTFT (ms):                           409.51    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          36.74     \nMedian TPOT (ms):                        35.30     \nP99 TPOT (ms):                           77.70     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           35.30     \nMedian ITL (ms):                         26.57     \nP99 ITL (ms):                            192.53    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 402.3294672546908,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 2.4855251265176648,
-    "tokens_per_second": 1830.1667164087519
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7edefddeafc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b23a48ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.20    \nTotal input tokens:                      38756     \nTotal generated tokens:                  39194     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.78    \nPeak output token throughput (tok/s):    495.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          423.18    \n---------------Time to First Token----------------\nMean TTFT (ms):                          60.23     \nMedian TTFT (ms):                        37.85     \nP99 TTFT (ms):                           146.84    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.21     \nMedian TPOT (ms):                        11.26     \nP99 TPOT (ms):                           18.07     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.00     \nMedian ITL (ms):                         10.95     \nP99 ITL (ms):                            19.93     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7400f615afc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2c7096a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.29    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151404    \nRequest throughput (req/s):              3.78      \nOutput token throughput (tok/s):         795.67    \nPeak output token throughput (tok/s):    1315.00   \nPeak concurrent requests:                36.00     \nTotal Token throughput (tok/s):          1560.51   \n---------------Time to First Token----------------\nMean TTFT (ms):                          65.82     \nMedian TTFT (ms):                        45.53     \nP99 TTFT (ms):                           177.98    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          20.36     \nMedian TPOT (ms):                        20.02     \nP99 TPOT (ms):                           31.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           20.10     \nMedian ITL (ms):                         18.54     \nP99 ITL (ms):                            90.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 357.56243500020355,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 2.7967143696161223,
-    "tokens_per_second": 2066.190202557434
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x71aab4572fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96888f07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  190.82    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         211.17    \nPeak output token throughput (tok/s):    443.00    \nPeak concurrent requests:                11.00     \nTotal Token throughput (tok/s):          412.18    \n---------------Time to First Token----------------\nMean TTFT (ms):                          64.41     \nMedian TTFT (ms):                        50.82     \nP99 TTFT (ms):                           144.55    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          21.14     \nMedian TPOT (ms):                        20.98     \nP99 TPOT (ms):                           23.59     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           21.12     \nMedian ITL (ms):                         20.65     \nP99 ITL (ms):                            29.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76b9330fafc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bd32e896-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  194.55    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155576    \nRequest throughput (req/s):              3.70      \nOutput token throughput (tok/s):         799.66    \nPeak output token throughput (tok/s):    1300.00   \nPeak concurrent requests:                43.00     \nTotal Token throughput (tok/s):          1553.66   \n---------------Time to First Token----------------\nMean TTFT (ms):                          92.77     \nMedian TTFT (ms):                        68.07     \nP99 TTFT (ms):                           484.98    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          25.78     \nMedian TPOT (ms):                        25.52     \nP99 TPOT (ms):                           34.37     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           25.65     \nMedian ITL (ms):                         23.39     \nP99 ITL (ms):                            86.03     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 889.9994421191514,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 1.1235962099245025,
-    "tokens_per_second": 832.9600726881711
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7236816eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4cdb5c8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.35    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39718     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.63    \nPeak output token throughput (tok/s):    666.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          425.84    \n---------------Time to First Token----------------\nMean TTFT (ms):                          37.48     \nMedian TTFT (ms):                        30.33     \nP99 TTFT (ms):                           77.17     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.36      \nMedian TPOT (ms):                        7.11      \nP99 TPOT (ms):                           11.45     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.26      \nMedian ITL (ms):                         6.53      \nP99 ITL (ms):                            11.05     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73a294576fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-81c0edc1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.20    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153033    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         821.89    \nPeak output token throughput (tok/s):    1471.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1609.74   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.36     \nMedian TTFT (ms):                        42.14     \nP99 TTFT (ms):                           82.58     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          13.89     \nMedian TPOT (ms):                        13.91     \nP99 TPOT (ms):                           17.75     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           13.74     \nMedian ITL (ms):                         13.29     \nP99 ITL (ms):                            38.23     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 215.380199868232,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.642952326220295,
-    "tokens_per_second": 3441.9784198061966
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x76db7ad72fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-04789567-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  188.87    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.95      \nOutput token throughput (tok/s):         205.96    \nPeak output token throughput (tok/s):    465.00    \nPeak concurrent requests:                10.00     \nTotal Token throughput (tok/s):          406.32    \n---------------Time to First Token----------------\nMean TTFT (ms):                          52.16     \nMedian TTFT (ms):                        44.25     \nP99 TTFT (ms):                           112.79    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          17.68     \nMedian TPOT (ms):                        17.62     \nP99 TPOT (ms):                           19.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           17.64     \nMedian ITL (ms):                         17.35     \nP99 ITL (ms):                            23.66     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7b8c3837efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b098e8e8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  193.10    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152091    \nRequest throughput (req/s):              3.73      \nOutput token throughput (tok/s):         787.62    \nPeak output token throughput (tok/s):    1351.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1542.71   \n---------------Time to First Token----------------\nMean TTFT (ms):                          57.32     \nMedian TTFT (ms):                        49.20     \nP99 TTFT (ms):                           126.64    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          20.27     \nMedian TPOT (ms):                        20.07     \nP99 TPOT (ms):                           27.95     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           20.15     \nMedian ITL (ms):                         19.30     \nP99 ITL (ms):                            62.76     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 252.42282237578183,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 3.9616069204364575,
-    "tokens_per_second": 2917.0500237249767
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7304080eefc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31ba87dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.12    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38869     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.27    \nPeak output token throughput (tok/s):    609.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.91    \n---------------Time to First Token----------------\nMean TTFT (ms):                          40.22     \nMedian TTFT (ms):                        36.83     \nP99 TTFT (ms):                           74.88     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.07      \nMedian TPOT (ms):                        8.23      \nP99 TPOT (ms):                           11.78     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.04      \nMedian ITL (ms):                         8.17      \nP99 ITL (ms):                            12.73     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7cdc7797efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b94205ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  187.07    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151811    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         811.50    \nPeak output token throughput (tok/s):    1549.00   \nPeak concurrent requests:                30.00     \nTotal Token throughput (tok/s):          1589.49   \n---------------Time to First Token----------------\nMean TTFT (ms):                          41.02     \nMedian TTFT (ms):                        36.91     \nP99 TTFT (ms):                           79.84     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.71     \nMedian TPOT (ms):                        14.74     \nP99 TPOT (ms):                           17.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.60     \nMedian ITL (ms):                         14.44     \nP99 ITL (ms):                            32.49     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 275.10345687624067,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 3.6349961260205634,
-    "tokens_per_second": 2685.5060579349843
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f90e18fd9e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0abaed61-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  192.06    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.94      \nOutput token throughput (tok/s):         209.81    \nPeak output token throughput (tok/s):    440.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          409.54    \n---------------Time to First Token----------------\nMean TTFT (ms):                          51.40     \nMedian TTFT (ms):                        47.89     \nP99 TTFT (ms):                           93.19     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          23.06     \nMedian TPOT (ms):                        23.00     \nP99 TPOT (ms):                           24.12     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           23.10     \nMedian ITL (ms):                         22.83     \nP99 ITL (ms):                            28.65     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f205d7019e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-14719fc9-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  195.24    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155585    \nRequest throughput (req/s):              3.69      \nOutput token throughput (tok/s):         796.89    \nPeak output token throughput (tok/s):    1321.00   \nPeak concurrent requests:                39.00     \nTotal Token throughput (tok/s):          1548.24   \n---------------Time to First Token----------------\nMean TTFT (ms):                          53.18     \nMedian TTFT (ms):                        49.55     \nP99 TTFT (ms):                           101.04    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          24.41     \nMedian TPOT (ms):                        24.26     \nP99 TPOT (ms):                           28.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           24.37     \nMedian ITL (ms):                         23.61     \nP99 ITL (ms):                            56.70     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 228.03156735002995,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.385357745074801,
-    "tokens_per_second": 3251.0147985872827
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbbe1e19e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5741d2ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.76    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39611     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         216.74    \nPeak output token throughput (tok/s):    789.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          426.61    \n---------------Time to First Token----------------\nMean TTFT (ms):                          30.25     \nMedian TTFT (ms):                        23.16     \nP99 TTFT (ms):                           57.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.38      \nMedian TPOT (ms):                        5.32      \nP99 TPOT (ms):                           7.38      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.36      \nMedian ITL (ms):                         5.29      \nP99 ITL (ms):                            7.43      \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6e920219e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4730dd07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.59    \nTotal input tokens:                      146694    \nTotal generated tokens:                  152861    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         832.64    \nPeak output token throughput (tok/s):    1713.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1631.69   \n---------------Time to First Token----------------\nMean TTFT (ms):                          29.84     \nMedian TTFT (ms):                        24.09     \nP99 TTFT (ms):                           60.04     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.01      \nMedian TPOT (ms):                        7.77      \nP99 TPOT (ms):                           11.35     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.96      \nMedian ITL (ms):                         7.28      \nP99 ITL (ms):                            27.43     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 133.56136341392994,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 7.487195207051202,
-    "tokens_per_second": 5550.512371624096
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fcbc7235c60>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d6b41380-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  185.59    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         209.60    \nPeak output token throughput (tok/s):    517.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          413.50    \n---------------Time to First Token----------------\nMean TTFT (ms):                          39.73     \nMedian TTFT (ms):                        32.58     \nP99 TTFT (ms):                           92.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.35     \nMedian TPOT (ms):                        11.23     \nP99 TPOT (ms):                           13.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.32     \nMedian ITL (ms):                         11.05     \nP99 ITL (ms):                            13.41     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5c44aa9c60>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-10085502-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  188.38    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152175    \nRequest throughput (req/s):              3.82      \nOutput token throughput (tok/s):         807.79    \nPeak output token throughput (tok/s):    1510.00   \nPeak concurrent requests:                29.00     \nTotal Token throughput (tok/s):          1581.79   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.75     \nMedian TTFT (ms):                        31.18     \nP99 TTFT (ms):                           92.33     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.77     \nMedian TPOT (ms):                        12.43     \nP99 TPOT (ms):                           17.06     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.71     \nMedian ITL (ms):                         11.91     \nP99 ITL (ms):                            36.70     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 230.30906111001968,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 4.341991562035397,
-    "tokens_per_second": 3197.1386468735236
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c0eab59e0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7ca49d8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  182.22    \nTotal input tokens:                      38756     \nTotal generated tokens:                  39194     \nRequest throughput (req/s):              0.99      \nOutput token throughput (tok/s):         215.09    \nPeak output token throughput (tok/s):    788.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          427.78    \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.17     \nMedian TTFT (ms):                        21.35     \nP99 TTFT (ms):                           52.39     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          5.08      \nMedian TPOT (ms):                        5.07      \nP99 TPOT (ms):                           7.44      \n---------------Inter-token Latency----------------\nMean ITL (ms):                           5.10      \nMedian ITL (ms):                         5.12      \nP99 ITL (ms):                            7.77      \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8d131e19e0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1904b29f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  183.65    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151340    \nRequest throughput (req/s):              3.92      \nOutput token throughput (tok/s):         824.06    \nPeak output token throughput (tok/s):    1801.00   \nPeak concurrent requests:                25.00     \nTotal Token throughput (tok/s):          1616.54   \n---------------Time to First Token----------------\nMean TTFT (ms):                          26.49     \nMedian TTFT (ms):                        22.28     \nP99 TTFT (ms):                           56.70     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.24      \nMedian TPOT (ms):                        8.09      \nP99 TPOT (ms):                           10.51     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.19      \nMedian ITL (ms):                         7.72      \nP99 ITL (ms):                            14.51     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 107.1224673166871,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 9.335109851826799,
-    "tokens_per_second": 6896.704477650824
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f88cf679990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-66e0ceca-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.88    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40286     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         214.42    \nPeak output token throughput (tok/s):    460.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          418.59    \n---------------Time to First Token----------------\nMean TTFT (ms):                          77.11     \nMedian TTFT (ms):                        52.45     \nP99 TTFT (ms):                           199.22    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          15.76     \nMedian TPOT (ms):                        15.43     \nP99 TPOT (ms):                           21.30     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.60     \nMedian ITL (ms):                         14.96     \nP99 ITL (ms):                            17.95     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f719e189990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-702c15c4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.21    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155632    \nRequest throughput (req/s):              3.79      \nOutput token throughput (tok/s):         818.19    \nPeak output token throughput (tok/s):    1520.00   \nPeak concurrent requests:                37.00     \nTotal Token throughput (tok/s):          1589.40   \n---------------Time to First Token----------------\nMean TTFT (ms):                          76.15     \nMedian TTFT (ms):                        45.58     \nP99 TTFT (ms):                           247.34    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          19.54     \nMedian TPOT (ms):                        18.83     \nP99 TPOT (ms):                           35.16     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           19.12     \nMedian ITL (ms):                         16.70     \nP99 ITL (ms):                            114.66    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 248.4086626805365,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 4.0256245060424485,
-    "tokens_per_second": 2984.3323175624723
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc3c8c05990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b80b1bb5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.14    \nTotal input tokens:                      38358     \nTotal generated tokens:                  39211     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         212.94    \nPeak output token throughput (tok/s):    560.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          421.26    \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.12     \nMedian TTFT (ms):                        41.56     \nP99 TTFT (ms):                           82.51     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          8.89      \nMedian TPOT (ms):                        8.73      \nP99 TPOT (ms):                           13.87     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           8.70      \nMedian ITL (ms):                         8.44      \nP99 ITL (ms):                            11.78     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ff610411990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fb44348e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  186.80    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153333    \nRequest throughput (req/s):              3.85      \nOutput token throughput (tok/s):         820.85    \nPeak output token throughput (tok/s):    1449.00   \nPeak concurrent requests:                31.00     \nTotal Token throughput (tok/s):          1606.16   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.68     \nMedian TTFT (ms):                        36.77     \nP99 TTFT (ms):                           89.21     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.46     \nMedian TPOT (ms):                        14.20     \nP99 TPOT (ms):                           19.40     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         13.63     \nP99 ITL (ms):                            44.44     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 199.3360565919429,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 5.016653871341908,
-    "tokens_per_second": 3719.016081057382
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fe9a3b39990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4225f4cd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  187.19    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38830     \nRequest throughput (req/s):              0.96      \nOutput token throughput (tok/s):         207.43    \nPeak output token throughput (tok/s):    490.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          409.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          45.84     \nMedian TTFT (ms):                        37.87     \nP99 TTFT (ms):                           101.24    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.35     \nMedian TPOT (ms):                        14.27     \nP99 TPOT (ms):                           16.03     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.28     \nMedian ITL (ms):                         14.04     \nP99 ITL (ms):                            16.81     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f45831f1990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ca67e7d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  190.55    \nTotal input tokens:                      145810    \nTotal generated tokens:                  152171    \nRequest throughput (req/s):              3.78      \nOutput token throughput (tok/s):         798.60    \nPeak output token throughput (tok/s):    1492.00   \nPeak concurrent requests:                35.00     \nTotal Token throughput (tok/s):          1563.82   \n---------------Time to First Token----------------\nMean TTFT (ms):                          44.19     \nMedian TTFT (ms):                        37.54     \nP99 TTFT (ms):                           101.54    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          16.05     \nMedian TPOT (ms):                        15.69     \nP99 TPOT (ms):                           20.99     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           15.95     \nMedian ITL (ms):                         15.02     \nP99 ITL (ms):                            41.81     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 195.21851312741637,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 5.122464995660089,
-    "tokens_per_second": 3771.8246502543934
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc0cf5b5990>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-85d6e157-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  183.05    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38779     \nRequest throughput (req/s):              0.98      \nOutput token throughput (tok/s):         211.85    \nPeak output token throughput (tok/s):    676.00    \nPeak concurrent requests:                8.00      \nTotal Token throughput (tok/s):          423.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          36.19     \nMedian TTFT (ms):                        30.20     \nP99 TTFT (ms):                           83.37     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          7.40      \nMedian TPOT (ms):                        7.32      \nP99 TPOT (ms):                           10.60     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           7.34      \nMedian ITL (ms):                         7.07      \nP99 ITL (ms):                            10.69     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6542301990>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-584a872e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  185.93    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151690    \nRequest throughput (req/s):              3.87      \nOutput token throughput (tok/s):         815.86    \nPeak output token throughput (tok/s):    1630.00   \nPeak concurrent requests:                28.00     \nTotal Token throughput (tok/s):          1598.63   \n---------------Time to First Token----------------\nMean TTFT (ms):                          38.36     \nMedian TTFT (ms):                        31.91     \nP99 TTFT (ms):                           78.67     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          12.67     \nMedian TPOT (ms):                        12.82     \nP99 TPOT (ms):                           16.44     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           12.51     \nMedian ITL (ms):                         12.62     \nP99 ITL (ms):                            31.79     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 156.32366928458214,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 6.396983928131399,
-    "tokens_per_second": 4726.040550232053
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7d8533bc6fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-39a9e102-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  197.48    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40296     \nRequest throughput (req/s):              0.91      \nOutput token throughput (tok/s):         204.06    \nPeak output token throughput (tok/s):    372.00    \nPeak concurrent requests:                15.00     \nTotal Token throughput (tok/s):          398.30    \n---------------Time to First Token----------------\nMean TTFT (ms):                          78.01     \nMedian TTFT (ms):                        70.55     \nP99 TTFT (ms):                           149.81    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          34.15     \nMedian TPOT (ms):                        34.06     \nP99 TPOT (ms):                           36.67     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           34.16     \nMedian ITL (ms):                         33.55     \nP99 ITL (ms):                            58.95     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ad99aa1efc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0e099965-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  205.53    \nTotal input tokens:                      146694    \nTotal generated tokens:                  155647    \nRequest throughput (req/s):              3.50      \nOutput token throughput (tok/s):         757.31    \nPeak output token throughput (tok/s):    1260.00   \nPeak concurrent requests:                59.00     \nTotal Token throughput (tok/s):          1471.06   \n---------------Time to First Token----------------\nMean TTFT (ms):                          85.03     \nMedian TTFT (ms):                        76.62     \nP99 TTFT (ms):                           176.99    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          41.66     \nMedian TPOT (ms):                        41.32     \nP99 TPOT (ms):                           52.93     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           41.47     \nMedian ITL (ms):                         39.19     \nP99 ITL (ms):                            111.74    \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 405.9800275520247,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 2.463175358723414,
-    "tokens_per_second": 1826.0356413838635
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x74f226dff240>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-341eb0ef-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  184.76    \nTotal input tokens:                      38358     \nTotal generated tokens:                  40087     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         216.97    \nPeak output token throughput (tok/s):    453.00    \nPeak concurrent requests:                9.00      \nTotal Token throughput (tok/s):          424.58    \n---------------Time to First Token----------------\nMean TTFT (ms):                          40.37     \nMedian TTFT (ms):                        37.70     \nP99 TTFT (ms):                           70.13     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          11.50     \nMedian TPOT (ms):                        11.56     \nP99 TPOT (ms):                           16.13     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           11.24     \nMedian ITL (ms):                         11.23     \nP99 ITL (ms):                            19.54     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x73d7f5083240>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-34321bd4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  192.26    \nTotal input tokens:                      146694    \nTotal generated tokens:                  153536    \nRequest throughput (req/s):              3.74      \nOutput token throughput (tok/s):         798.60    \nPeak output token throughput (tok/s):    1194.00   \nPeak concurrent requests:                46.00     \nTotal Token throughput (tok/s):          1561.61   \n---------------Time to First Token----------------\nMean TTFT (ms):                          54.49     \nMedian TTFT (ms):                        52.17     \nP99 TTFT (ms):                           97.75     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          25.72     \nMedian TPOT (ms):                        25.55     \nP99 TPOT (ms):                           31.80     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           25.55     \nMedian ITL (ms):                         24.86     \nP99 ITL (ms):                            53.01     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 307.8667154959985,
-    "num_requests": 1000,
-    "total_num_tokens": 741334,
-    "requests_per_second": 3.248158861177695,
-    "tokens_per_second": 2407.9706011923054
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7bbdd0516fc0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f41559c3-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  195.60    \nTotal input tokens:                      37841     \nTotal generated tokens:                  38900     \nRequest throughput (req/s):              0.92      \nOutput token throughput (tok/s):         198.88    \nPeak output token throughput (tok/s):    395.00    \nPeak concurrent requests:                13.00     \nTotal Token throughput (tok/s):          392.34    \n---------------Time to First Token----------------\nMean TTFT (ms):                          63.33     \nMedian TTFT (ms):                        59.42     \nP99 TTFT (ms):                           123.97    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          30.79     \nMedian TPOT (ms):                        30.76     \nP99 TPOT (ms):                           32.51     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           30.78     \nMedian ITL (ms):                         30.39     \nP99 ITL (ms):                            42.55     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7b7e08676fc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f456f8a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     719       \nFailed requests:                         1         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  204.29    \nTotal input tokens:                      145785    \nTotal generated tokens:                  151865    \nRequest throughput (req/s):              3.52      \nOutput token throughput (tok/s):         743.36    \nPeak output token throughput (tok/s):    1197.00   \nPeak concurrent requests:                52.00     \nTotal Token throughput (tok/s):          1456.96   \n---------------Time to First Token----------------\nMean TTFT (ms):                          68.23     \nMedian TTFT (ms):                        63.70     \nP99 TTFT (ms):                           136.63    \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          34.84     \nMedian TPOT (ms):                        34.66     \nP99 TPOT (ms):                           41.64     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           34.70     \nMedian ITL (ms):                         33.44     \nP99 ITL (ms):                            79.44     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 342.5061867629993,
-    "num_requests": 1000,
-    "total_num_tokens": 736330,
-    "requests_per_second": 2.9196552898822827,
-    "tokens_per_second": 2149.8297795990215
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps1.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fe04afbf240>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-68841ca7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     180       \nFailed requests:                         0         \nRequest rate configured (RPS):           1.00      \nBenchmark duration (s):                  185.44    \nTotal input tokens:                      38756     \nTotal generated tokens:                  38777     \nRequest throughput (req/s):              0.97      \nOutput token throughput (tok/s):         209.11    \nPeak output token throughput (tok/s):    437.00    \nPeak concurrent requests:                10.00     \nTotal Token throughput (tok/s):          418.10    \n---------------Time to First Token----------------\nMean TTFT (ms):                          42.77     \nMedian TTFT (ms):                        40.02     \nP99 TTFT (ms):                           73.37     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          14.89     \nMedian TPOT (ms):                        14.86     \nP99 TPOT (ms):                           21.46     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           14.57     \nMedian ITL (ms):                         14.54     \nP99 ITL (ms):                            24.86     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json

@@ -1,4 +0,0 @@
-{
-  "success": true,
-  "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x75d3437befc0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96562708-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests:                     720       \nFailed requests:                         0         \nRequest rate configured (RPS):           4.00      \nBenchmark duration (s):                  194.32    \nTotal input tokens:                      145540    \nTotal generated tokens:                  151349    \nRequest throughput (req/s):              3.71      \nOutput token throughput (tok/s):         778.86    \nPeak output token throughput (tok/s):    1218.00   \nPeak concurrent requests:                42.00     \nTotal Token throughput (tok/s):          1527.82   \n---------------Time to First Token----------------\nMean TTFT (ms):                          51.01     \nMedian TTFT (ms):                        49.88     \nP99 TTFT (ms):                           83.96     \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms):                          26.15     \nMedian TPOT (ms):                        26.20     \nP99 TPOT (ms):                           28.93     \n---------------Inter-token Latency----------------\nMean ITL (ms):                           26.04     \nMedian ITL (ms):                         25.85     \nP99 ITL (ms):                            42.96     \n==================================================\n"
-}

benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_throughput.json

@@ -1,7 +0,0 @@
-{
-    "elapsed_time": 256.2449242460134,
-    "num_requests": 1000,
-    "total_num_tokens": 738792,
-    "requests_per_second": 3.9025163247328507,
-    "tokens_per_second": 2883.1478405820326
-}

benchmarks/run_vllm_bench_nvidia.py

@@ -1,392 +0,0 @@
-#!/usr/bin/env python3
-import subprocess, time, json, sys, os, requests, re, argparse
-from pathlib import Path
-
-# =========================
-# ⚙️ GLOBAL SETTINGS
-# =========================
-
-# HARDWARE: NVIDIA GPUs (Auto-detected)
-GPU_UTIL = "0.95" 
-PORT     = 8000
-HOST     = "127.0.0.1"
-
-# 1. THROUGHPUT CONFIG
-OFF_NUM_PROMPTS      = 1000 
-OFF_FORCED_OUTPUT    = "512"
-# Default fallback if not specified in MODEL_TABLE
-DEFAULT_BATCH_TOKENS = "8192"
-
-# 2. LATENCY CONFIG
-SRV_DURATION    = 180    
-QPS_SWEEP       = [1.0, 4.0] 
-
-# Fallbacks
-FALLBACK_INPUT_LEN  = 1024
-FALLBACK_OUTPUT_LEN = 512
-
-RESULTS_DIR = Path("benchmark_results_nvidia")
-RESULTS_DIR.mkdir(exist_ok=True)
-
-# =========================
-# 🛠️ MODEL CONFIGURATION 🛠️
-# =========================
-
-MODEL_TABLE = {
-    # 1. Llama 3.1 8B Instruct
-    "meta-llama/Meta-Llama-3.1-8B-Instruct": {
-        "ctx": "65536",  
-        "trust_remote": False,
-        "valid_tp": [1, 2],
-        "max_num_seqs": "64",
-        "max_tokens": "32768" 
-    },
-    
-    # 2. GPT-OSS 20B (MXFP4)
-    "openai/gpt-oss-20b": {
-        "ctx": "32768", 
-        "trust_remote": True,
-        "valid_tp": [1, 2],
-        "max_num_seqs": "64",
-        "max_tokens": "8192",
-    },
-
-    # 3. Qwen 14B FP8
-    "RedHatAI/Qwen3-14B-FP8-dynamic": {
-        "ctx": "32768", 
-        "trust_remote": True,
-        "valid_tp": [1],
-        "max_num_seqs": "64",
-        "max_tokens": "32768",
-        "gpu_util": "0.90"
-    },
-
-    # 4. Qwen 30B 4-bit
-    "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
-        "ctx": "24576", 
-        "trust_remote": True,
-        "valid_tp": [1, 2],
-        "max_num_seqs": "64",
-        "max_tokens": "32768",
-        "gpu_util": "0.90"
-    },
-
-    # 5. Qwen 80B AWQ
-    "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
-        "ctx": "20480", 
-        "trust_remote": True,
-        "valid_tp": [2], # Requires 2 GPUs
-        "max_num_seqs": "32", 
-        "max_tokens": "16384",
-    },
-
-    # 6. Llama 3.1 8B FP8
-    "RedHatAI/Llama-3.1-8B-Instruct-FP8-block": {
-        "ctx": "65536",
-        "trust_remote": True,
-        "valid_tp": [1, 2],
-        "max_num_seqs": "64",
-        "max_tokens": "32768",
-    },
-
-    # 7. Gemma 3 12B FP8
-    "RedHatAI/gemma-3-12b-it-FP8-dynamic": {
-        "ctx": "32768",
-        "trust_remote": True,
-        "valid_tp": [1, 2],
-        "max_num_seqs": "64",
-        "max_tokens": "32768",
-    },
-}
-
-MODELS_TO_RUN = [
-    "meta-llama/Meta-Llama-3.1-8B-Instruct",
-    "openai/gpt-oss-20b",
-    "RedHatAI/Qwen3-14B-FP8-dynamic",
-    "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
-    "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
-    "RedHatAI/gemma-3-12b-it-FP8-dynamic",
-]
-
-# =========================
-# UTILS
-# =========================
-
-def log(msg): print(f"\n[BENCH] {msg}")
-
-def get_gpu_count():
-    try:
-        # Using nvidia-smi -L to list GPUs
-        res = subprocess.run(["nvidia-smi", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        if res.returncode == 0:
-            count = len([line for line in res.stdout.strip().split('\n') if line.strip()])
-            return count
-        else:
-            log("nvidia-smi failed, defaulting to 1 GPU")
-            return 1
-    except Exception as e:
-        log(f"Error detecting GPUs: {e}, defaulting to 1 GPU")
-        return 1
-
-def force_gpu_cleanup():
-    """Simple cleanup: just kill vllm processes (excluding self)."""
-    try:
-        my_pid = os.getpid()
-        # Kill everything matching vllm EXCEPT this process
-        subprocess.run(f"pgrep -f 'vllm' | grep -v {my_pid} | xargs -r kill -9", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        # Original cleanups for other helpers
-        subprocess.run("pkill -9 -f 'multiprocessing'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        subprocess.run("pkill -9 -f 'resource_tracker'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        
-        # Try finding fuser to kill processes attached to device files
-        if subprocess.run("which fuser", shell=True, stdout=subprocess.DEVNULL).returncode == 0:
-             subprocess.run("fuser -k -9 /dev/nvidia*", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    except: pass
-    time.sleep(5)
-
-def nuke_vllm_cache():
-    cache = Path.home() / ".cache" / "vllm"
-    if cache.exists():
-        try:
-            subprocess.run(["rm", "-rf", str(cache)], check=True)
-            cache.mkdir(parents=True, exist_ok=True)
-            time.sleep(2)
-        except: pass
-
-def get_dataset():
-    data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json")
-    if data_path.exists(): return str(data_path)
-    
-    log("Downloading ShareGPT dataset...")
-    url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
-    try:
-        r = requests.get(url, stream=True, timeout=15)
-        r.raise_for_status()
-        with open(data_path, 'wb') as f:
-            for chunk in r.iter_content(chunk_size=8192): f.write(chunk)
-        return str(data_path)
-    except Exception as e:
-        log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.")
-        return None
-
-def wait_for_server(url, process, timeout=600):
-    start = time.time()
-    while time.time() - start < timeout:
-        if process.poll() is not None:
-            log(f"CRITICAL: Server died! Ret: {process.returncode}")
-            return False
-        try:
-            if requests.get(f"{url}/v1/models", timeout=2).status_code == 200:
-                log("Server ready. Stabilizing...")
-                time.sleep(5)
-                return True
-        except: pass
-        time.sleep(2)
-    return False
-
-# =========================
-# HARDWARE DETECTION (24GB vs 32GB)
-# =========================
-def is_24gb_card():
-    try:
-        res = subprocess.run(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], 
-                             capture_output=True, text=True)
-        # Check first GPU memory
-        mem = int(res.stdout.strip().split('\n')[0])
-        return mem < 28000 # 4090 is ~24576
-    except:
-        return False
-
-IS_24GB = is_24gb_card()
-if IS_24GB: log("Detected 24GB GPU class (e.g. RTX 4090). Applying memory overrides.")
-else: log("Detected 32GB+ GPU class. Using standard config.")
-
-def get_model_args(model, tp_size):
-    config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"})
-    
-    current_ctx = config["ctx"]
-    current_seqs = config["max_num_seqs"]
-    util = None
-
-    if IS_24GB:
-        if model == "meta-llama/Meta-Llama-3.1-8B-Instruct":
-            current_ctx = "31800"
-            log(f"Override: Llama 8B ctx reduced to {current_ctx} for 24GB VRAM")
-        elif model == "openai/gpt-oss-20b":
-            current_ctx = "16384"
-            current_seqs = "32"
-            util = "0.90"
-            log(f"Override: GPT-20B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM")
-        elif model == "RedHatAI/Qwen3-14B-FP8-dynamic":
-            current_ctx = "4096"
-            current_seqs = "32"
-            util = "0.86"
-            log(f"Override: Qwen 14B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM")
-    
-    if util is None: util = config.get("gpu_util", GPU_UTIL)
-    
-    cmd = [
-        "--model", model,
-        "--gpu-memory-utilization", util,
-        "--max-model-len", current_ctx, 
-        "--dtype", "auto",
-        "--tensor-parallel-size", str(tp_size),
-        "--max-num-seqs", current_seqs
-    ]
-    
-    if config.get("trust_remote"): cmd.append("--trust-remote-code")
-    if config.get("enforce_eager"): cmd.append("--enforce-eager")
-    
-    return cmd
-
-def run_throughput(model, tp_size):
-    if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
-    
-    model_safe = model.replace("/", "_")
-    output_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_throughput.json"
-    
-    if output_file.exists():
-        log(f"SKIP Throughput {model} (TP={tp_size})")
-        return
-
-    dataset_path = get_dataset()
-    dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"]
-    
-    batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS)
-
-    log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...")
-    force_gpu_cleanup()
-    nuke_vllm_cache()
-
-    cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size)
-    cmd.extend([
-        "--num-prompts", str(OFF_NUM_PROMPTS),
-        "--max-num-batched-tokens", batch_tokens,
-        "--output-len", OFF_FORCED_OUTPUT,
-        "--output-json", str(output_file),
-        "--disable-log-stats"
-    ])
-    cmd.extend(dataset_args)
-
-    env = os.environ.copy()
-    env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
-    model_env = MODEL_TABLE[model].get("env", {})
-    env.update(model_env)
-
-    ids_cmd = " ".join(cmd)
-    log(f"CMD: {ids_cmd}")
-
-    try: 
-        subprocess.run(cmd, check=True, env=env)
-    except: 
-        log(f"ERROR: Throughput failed {model}")
-
-def run_latency(model, tp_size):
-    if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
-    model_safe = model.replace("/", "_")
-
-    if all((RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{q}_latency.json").exists() for q in QPS_SWEEP):
-        return
-
-    dataset_path = get_dataset()
-    log(f"START Server {model} (TP={tp_size})...")
-    force_gpu_cleanup()
-    nuke_vllm_cache()
-
-
-    srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w")
-    
-    # Use get_model_args directly. It includes ["--model", model, ...]
-    srv_args = get_model_args(model, tp_size) 
-    
-    env = os.environ.copy()
-    env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True"
-    model_env = MODEL_TABLE[model].get("env", {})
-    env.update(model_env)
-
-    # Command: vllm serve --model <model> ... (no positional model arg)
-    cmd = ["vllm", "serve"] + srv_args + ["--host", HOST, "--port", str(PORT)]
-    
-    ids_cmd = " ".join(cmd)
-    log(f"CMD: {ids_cmd}")
-
-    proc = subprocess.Popen(cmd, stdout=srv_log, stderr=srv_log, env=env)
-
-    try:
-        if not wait_for_server(f"http://{HOST}:{PORT}", proc): return
-
-        for qps in QPS_SWEEP:
-            out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json"
-            if out_file.exists(): continue
-            
-            log(f"BENCH QPS={qps}...")
-            bench_cmd = [
-                "vllm", "bench", "serve",
-                "--model", model,
-                "--base-url", f"http://{HOST}:{PORT}",
-                "--request-rate", str(qps),
-                "--num-prompts", str(int(max(10, SRV_DURATION * qps))),
-                "--trust-remote-code"
-            ]
-            
-            if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path])
-            else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"])
-
-            res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env)
-            with open(out_file, "w") as f:
-                f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2))
-
-    except Exception as e: log(f"CRASH: {e}")
-    finally:
-        proc.terminate()
-        force_gpu_cleanup()
-
-def print_summary(tps):
-    print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}")
-    print("-" * 105)
-    
-    for m in MODELS_TO_RUN:
-        msafe = m.replace("/", "_")
-        for tp in tps:
-            if tp not in MODEL_TABLE[m]["valid_tp"]: continue
-            
-            try: 
-                tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text())
-                tok_s = f"{tdata.get('tokens_per_second', 0):.1f}"
-            except: tok_s = "N/A"
-
-            first_row = True
-            for q in QPS_SWEEP:
-                try:
-                    ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text())
-                    raw = ldata["raw_output"]
-                    ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1)
-                    tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1)
-                except: ttft, tpot = "-", "-"
-                
-                name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else ""
-                
-                print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}")
-                first_row = False
-            print("-" * 105)
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--tp", type=int, nargs="+", default=[1, 2])
-    args = parser.parse_args()
-    
-    gpu_count = get_gpu_count()
-    log(f"Detected {gpu_count} GPU(s)")
-    
-    valid_tp_args = [t for t in args.tp if t <= gpu_count]
-    if not valid_tp_args:
-        log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.")
-        sys.exit(0)
-
-    force_gpu_cleanup()
-    for tp in valid_tp_args:
-        for m in MODELS_TO_RUN:
-            run_throughput(m, tp)
-            run_latency(m, tp)
-    print_summary(valid_tp_args)