4 lines
3.6 KiB
JSON
4 lines
3.6 KiB
JSON
{
|
|
"success": true,
|
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f83b1b0aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f36abb16-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 241.29 \nTotal input tokens: 158086 \nTotal generated tokens: 152183 \nRequest throughput (req/s): 2.98 \nOutput token throughput (tok/s): 630.70 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 152.00 \nTotal Token throughput (tok/s): 1285.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 12436.56 \nMedian TTFT (ms): 17115.90 \nP99 TTFT (ms): 24750.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 84.04 \nMedian TPOT (ms): 85.32 \nP99 TPOT (ms): 115.82 \n---------------Inter-token Latency----------------\nMean ITL (ms): 83.72 \nMedian ITL (ms): 75.54 \nP99 ITL (ms): 261.14 \n==================================================\n"
|
|
} |