1023 lines
116 KiB
Plaintext
1023 lines
116 KiB
Plaintext
|
WARNING 12-09 20:20:45 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:45 [api_server.py:1772] vLLM API server version 0.12.0
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:45 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 16384, 'max_num_seqs': 32}
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:53 [model.py:637] Resolved architecture: GptOssForCausalLM
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m
|
||
|
|
Parse safetensors files: 0%| | 0/3 [00:00<?, ?it/s]
|
||
|
|
Parse safetensors files: 33%|███▎ | 1/3 [00:00<00:00, 3.67it/s]
|
||
|
|
Parse safetensors files: 100%|██████████| 3/3 [00:00<00:00, 10.96it/s]
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:54 [model.py:1750] Using max model len 16384
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:54 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:20:54 [config.py:274] Overriding max cuda graph capture size to 1024 for performance.
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:02 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='openai/gpt-oss-20b', speculative_config=None, tokenizer='openai/gpt-oss-20b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=mxfp4, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='openai_gptoss', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=openai/gpt-oss-20b, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>}, 'local_cache_dir': None}
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:02 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:56281 backend=nccl
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:02 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:03 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b...
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:03 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN']
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:03 [layer.py:379] Enabled separate cuda stream for MoE shared_experts
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:03 [mxfp4.py:162] Using Marlin backend
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00<?, ?it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 33% Completed | 1/3 [00:00<00:01, 1.73it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 67% Completed | 2/3 [00:01<00:00, 1.46it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:02<00:00, 1.43it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 3/3 [00:02<00:00, 1.46it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:06 [default_loader.py:308] Loading weights took 2.21 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m WARNING 12-09 20:21:06 [marlin_utils_fp4.py:226] Your GPU does not have native support for FP4 computation but FP4 quantization is being used. Weight-only FP4 compression will be used leveraging the Marlin kernel. This may degrade performance for compute-heavy workloads.
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:07 [gpu_model_runner.py:3549] Model loading took 13.7194 GiB memory and 3.694696 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:11 [backends.py:655] Using cache directory: /root/.cache/vllm/torch_compile_cache/e1e56bd73f/rank_0_0/backbone for vLLM's torch.compile
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:11 [backends.py:715] Dynamo bytecode transform time: 3.61 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:11 [backends.py:257] Cache the graph for dynamic shape for later use
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:18 [backends.py:288] Compiling a graph for dynamic shape takes 6.55 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:23 [monitor.py:34] torch.compile takes 10.16 s in total
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:24 [gpu_worker.py:359] Available KV cache memory: 7.11 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:24 [kv_cache_utils.py:1286] GPU KV cache size: 155,344 tokens
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:24 [kv_cache_utils.py:1291] Maximum concurrency for 16,384 tokens per request: 16.73x
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/83 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 2%|▏ | 2/83 [00:00<00:06, 13.40it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▍ | 4/83 [00:00<00:05, 13.62it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 7%|▋ | 6/83 [00:00<00:05, 13.82it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 10%|▉ | 8/83 [00:00<00:05, 13.98it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 10/83 [00:00<00:05, 14.23it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 14%|█▍ | 12/83 [00:00<00:04, 14.43it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 17%|█▋ | 14/83 [00:00<00:04, 14.69it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 19%|█▉ | 16/83 [00:01<00:04, 14.91it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 22%|██▏ | 18/83 [00:01<00:04, 15.27it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 24%|██▍ | 20/83 [00:01<00:04, 15.53it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 22/83 [00:01<00:03, 15.96it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▉ | 24/83 [00:01<00:03, 16.24it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 31%|███▏ | 26/83 [00:01<00:03, 16.67it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 34%|███▎ | 28/83 [00:01<00:03, 16.96it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▌ | 30/83 [00:01<00:03, 17.51it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▊ | 32/83 [00:02<00:02, 17.81it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 35/83 [00:02<00:02, 18.67it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 46%|████▌ | 38/83 [00:02<00:02, 19.44it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 49%|████▉ | 41/83 [00:02<00:02, 20.24it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 44/83 [00:02<00:01, 21.18it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 47/83 [00:02<00:01, 22.21it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 60%|██████ | 50/83 [00:02<00:01, 23.06it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▍ | 53/83 [00:02<00:01, 23.64it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 67%|██████▋ | 56/83 [00:03<00:01, 23.97it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 71%|███████ | 59/83 [00:03<00:00, 24.19it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 62/83 [00:03<00:00, 23.59it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 78%|███████▊ | 65/83 [00:03<00:00, 18.91it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 68/83 [00:03<00:00, 20.31it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▌ | 71/83 [00:03<00:00, 21.36it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 74/83 [00:03<00:00, 22.16it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 93%|█████████▎| 77/83 [00:04<00:00, 23.00it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 96%|█████████▋| 80/83 [00:04<00:00, 23.73it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 83/83 [00:04<00:00, 23.82it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 83/83 [00:04<00:00, 19.36it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m
|
||
|
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 29%|██▊ | 2/7 [00:00<00:00, 14.29it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 57%|█████▋ | 4/7 [00:00<00:00, 15.77it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 86%|████████▌ | 6/7 [00:00<00:00, 16.73it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 7/7 [00:00<00:00, 16.32it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:29 [gpu_model_runner.py:4466] Graph capturing finished in 5 secs, took 0.71 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=19724)[0;0m INFO 12-09 20:21:29 [core.py:254] init engine (profile, create kv cache, warmup model) took 22.31 seconds
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:31 [api_server.py:1520] Supported tasks: ['generate']
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m WARNING 12-09 20:21:32 [serving_responses.py:215] For gpt-oss, we ignore --enable-auto-tool-choice and always enable tool use.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [api_server.py:1847] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:38] Available routes are:
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /health, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /load, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /pause, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /resume, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /version, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /ping, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /ping, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /invocations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /metrics, Methods: GET
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /classify, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:37 [launcher.py:46] Route: /pooling, Methods: POST
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: Started server process [19409]
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: Waiting for application startup.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: Application startup complete.
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48002 - "GET /v1/models HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:21:57 [loggers.py:236] Engine 000: Avg prompt throughput: 1.2 tokens/s, Avg generation throughput: 11.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38370 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38370 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:07 [loggers.py:236] Engine 000: Avg prompt throughput: 114.9 tokens/s, Avg generation throughput: 220.4 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:54096 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:17 [loggers.py:236] Engine 000: Avg prompt throughput: 205.8 tokens/s, Avg generation throughput: 188.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:27 [loggers.py:236] Engine 000: Avg prompt throughput: 188.6 tokens/s, Avg generation throughput: 146.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:37 [loggers.py:236] Engine 000: Avg prompt throughput: 347.7 tokens/s, Avg generation throughput: 170.3 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:47 [loggers.py:236] Engine 000: Avg prompt throughput: 176.1 tokens/s, Avg generation throughput: 219.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48110 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:48118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:22:57 [loggers.py:236] Engine 000: Avg prompt throughput: 426.9 tokens/s, Avg generation throughput: 182.1 tokens/s, Running: 3 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:38368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:07 [loggers.py:236] Engine 000: Avg prompt throughput: 296.2 tokens/s, Avg generation throughput: 238.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50540 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:17 [loggers.py:236] Engine 000: Avg prompt throughput: 310.8 tokens/s, Avg generation throughput: 242.8 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50540 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:27 [loggers.py:236] Engine 000: Avg prompt throughput: 172.6 tokens/s, Avg generation throughput: 374.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.8%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50546 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:33396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:37 [loggers.py:236] Engine 000: Avg prompt throughput: 431.1 tokens/s, Avg generation throughput: 210.6 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50532 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:47 [loggers.py:236] Engine 000: Avg prompt throughput: 36.4 tokens/s, Avg generation throughput: 242.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:23:57 [loggers.py:236] Engine 000: Avg prompt throughput: 180.3 tokens/s, Avg generation throughput: 156.5 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45496 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45496 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:07 [loggers.py:236] Engine 000: Avg prompt throughput: 262.5 tokens/s, Avg generation throughput: 284.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45496 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50530 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45496 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:17 [loggers.py:236] Engine 000: Avg prompt throughput: 96.4 tokens/s, Avg generation throughput: 251.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56352 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:50514 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:45496 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:27 [loggers.py:236] Engine 000: Avg prompt throughput: 68.6 tokens/s, Avg generation throughput: 58.3 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56690 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56690 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56690 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:37 [loggers.py:236] Engine 000: Avg prompt throughput: 135.1 tokens/s, Avg generation throughput: 226.2 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56670 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:56674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:47 [loggers.py:236] Engine 000: Avg prompt throughput: 109.7 tokens/s, Avg generation throughput: 130.7 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60108 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60108 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60136 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:49018 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60108 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60108 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60124 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:60132 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:24:57 [loggers.py:236] Engine 000: Avg prompt throughput: 315.8 tokens/s, Avg generation throughput: 289.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:07 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 53.9 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:17 [loggers.py:236] Engine 000: Avg prompt throughput: 136.1 tokens/s, Avg generation throughput: 123.1 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.5%, Prefix cache hit rate: 3.1%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:27 [loggers.py:236] Engine 000: Avg prompt throughput: 1057.0 tokens/s, Avg generation throughput: 664.9 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 22.7%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:37 [loggers.py:236] Engine 000: Avg prompt throughput: 1047.4 tokens/s, Avg generation throughput: 872.7 tokens/s, Running: 16 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 35.2%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:47 [loggers.py:236] Engine 000: Avg prompt throughput: 910.3 tokens/s, Avg generation throughput: 993.1 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 43.1%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:25:57 [loggers.py:236] Engine 000: Avg prompt throughput: 409.8 tokens/s, Avg generation throughput: 722.1 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.0%, Prefix cache hit rate: 45.9%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:07 [loggers.py:236] Engine 000: Avg prompt throughput: 1011.1 tokens/s, Avg generation throughput: 999.9 tokens/s, Running: 16 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.8%, Prefix cache hit rate: 44.0%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:17 [loggers.py:236] Engine 000: Avg prompt throughput: 998.4 tokens/s, Avg generation throughput: 801.2 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 39.4%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:27 [loggers.py:236] Engine 000: Avg prompt throughput: 537.4 tokens/s, Avg generation throughput: 773.8 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 37.3%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:37 [loggers.py:236] Engine 000: Avg prompt throughput: 831.5 tokens/s, Avg generation throughput: 1031.0 tokens/s, Running: 22 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.2%, Prefix cache hit rate: 34.4%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:47 [loggers.py:236] Engine 000: Avg prompt throughput: 652.9 tokens/s, Avg generation throughput: 1092.6 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 32.4%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:32800 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:26:57 [loggers.py:236] Engine 000: Avg prompt throughput: 877.1 tokens/s, Avg generation throughput: 725.7 tokens/s, Running: 14 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 30.7%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40808 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41334 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57948 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57954 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:07 [loggers.py:236] Engine 000: Avg prompt throughput: 914.8 tokens/s, Avg generation throughput: 692.7 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 28.6%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:17 [loggers.py:236] Engine 000: Avg prompt throughput: 507.9 tokens/s, Avg generation throughput: 649.6 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 27.6%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:27 [loggers.py:236] Engine 000: Avg prompt throughput: 844.8 tokens/s, Avg generation throughput: 855.0 tokens/s, Running: 15 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 26.1%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:37 [loggers.py:236] Engine 000: Avg prompt throughput: 894.6 tokens/s, Avg generation throughput: 735.4 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 24.6%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:47 [loggers.py:236] Engine 000: Avg prompt throughput: 581.1 tokens/s, Avg generation throughput: 648.9 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 23.7%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO 12-09 20:27:57 [loggers.py:236] Engine 000: Avg prompt throughput: 732.9 tokens/s, Avg generation throughput: 834.7 tokens/s, Running: 12 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 22.7%
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41238 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41326 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55076 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41346 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:41362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40838 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:57942 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:55072 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:59762 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=19409)[0;0m INFO: 127.0.0.1:40784 - "POST /v1/completions HTTP/1.1" 200 OK
|