1044 lines
115 KiB
Plaintext
1044 lines
115 KiB
Plaintext
|
/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden.
|
||
|
|
Overriding a previously registered kernel for the same operator and the same dispatch key
|
||
|
|
operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor
|
||
|
|
registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926
|
||
|
|
dispatch key: ADInplaceOrView
|
||
|
|
previous kernel: no debug info
|
||
|
|
new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.)
|
||
|
|
self.m.impl(
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:09 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:09 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:13 [model.py:629] Resolved architecture: LlamaForCausalLM
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:13 [model.py:1755] Using max model len 65536
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:13 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
|
|
/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden.
|
||
|
|
Overriding a previously registered kernel for the same operator and the same dispatch key
|
||
|
|
operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor
|
||
|
|
registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926
|
||
|
|
dispatch key: ADInplaceOrView
|
||
|
|
previous kernel: no debug info
|
||
|
|
new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.)
|
||
|
|
self.m.impl(
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:17 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:17 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:51127 backend=nccl
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:17 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:18 [gpu_model_runner.py:3544] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct...
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:18 [rocm.py:320] Using Triton Attention backend on V1 engine.
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 5.39it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.58it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:02<00:00, 1.25it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.14it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.28it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:22 [default_loader.py:308] Loading weights took 3.13 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:23 [gpu_model_runner.py:3626] Model loading took 15.0586 GiB memory and 4.320186 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:25 [backends.py:616] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/dfa108d3b7/rank_0_0/backbone for vLLM's torch.compile
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:25 [backends.py:676] Dynamo bytecode transform time: 2.05 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:29 [backends.py:243] Cache the graph of compile range (1, 2048) for later use
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:39 [backends.py:260] Compiling a graph for compile range (1, 2048) takes 12.70 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:39 [monitor.py:34] torch.compile takes 14.75 s in total
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:41 [gpu_worker.py:364] Available KV cache memory: 12.73 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:41 [kv_cache_utils.py:1287] GPU KV cache size: 104,320 tokens
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:41 [kv_cache_utils.py:1292] Maximum concurrency for 65,536 tokens per request: 1.59x
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:00, 19.11it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:00<00:00, 20.72it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:00<00:00, 21.20it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:00<00:00, 21.61it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:00<00:00, 21.93it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:00<00:00, 22.32it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:00<00:00, 21.91it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m
|
||
|
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:00<00:00, 21.69it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:00<00:00, 23.00it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:00<00:00, 22.61it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 22.60it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:43 [gpu_model_runner.py:4548] Graph capturing finished in 2 secs, took 1.36 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=4113)[0;0m INFO 12-09 18:49:43 [core.py:256] init engine (profile, create kv cache, warmup model) took 20.45 seconds
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:45 [api_server.py:1099] Supported tasks: ['generate']
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m WARNING 12-09 18:49:45 [model.py:1581] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:45 [serving_responses.py:197] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:45 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:45 [serving_completion.py:73] Using default completion sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_p': 0.9}
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [api_server.py:1425] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:38] Available routes are:
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /pause, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /resume, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /metrics, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /health, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /load, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /version, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /ping, Methods: GET
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /ping, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /invocations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /classify, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:49:46 [launcher.py:46] Route: /pooling, Methods: POST
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: Started server process [3950]
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: Waiting for application startup.
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: Application startup complete.
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42160 - "GET /v1/models HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:06 [loggers.py:248] Engine 000: Avg prompt throughput: 41.7 tokens/s, Avg generation throughput: 36.0 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:16 [loggers.py:248] Engine 000: Avg prompt throughput: 135.3 tokens/s, Avg generation throughput: 115.7 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:26 [loggers.py:248] Engine 000: Avg prompt throughput: 183.3 tokens/s, Avg generation throughput: 183.7 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:36 [loggers.py:248] Engine 000: Avg prompt throughput: 278.0 tokens/s, Avg generation throughput: 182.3 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:46 [loggers.py:248] Engine 000: Avg prompt throughput: 191.1 tokens/s, Avg generation throughput: 177.5 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:50:56 [loggers.py:248] Engine 000: Avg prompt throughput: 365.3 tokens/s, Avg generation throughput: 206.6 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51336 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51322 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:06 [loggers.py:248] Engine 000: Avg prompt throughput: 347.5 tokens/s, Avg generation throughput: 230.3 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40502 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42754 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:16 [loggers.py:248] Engine 000: Avg prompt throughput: 167.3 tokens/s, Avg generation throughput: 197.0 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43742 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43746 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43746 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43742 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43780 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43780 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:26 [loggers.py:248] Engine 000: Avg prompt throughput: 376.3 tokens/s, Avg generation throughput: 226.3 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43742 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43746 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:36 [loggers.py:248] Engine 000: Avg prompt throughput: 223.3 tokens/s, Avg generation throughput: 285.7 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43780 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43752 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:46 [loggers.py:248] Engine 000: Avg prompt throughput: 276.0 tokens/s, Avg generation throughput: 304.8 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:51:56 [loggers.py:248] Engine 000: Avg prompt throughput: 36.1 tokens/s, Avg generation throughput: 219.8 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:57994 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58004 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:06 [loggers.py:248] Engine 000: Avg prompt throughput: 238.5 tokens/s, Avg generation throughput: 176.6 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:38000 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:16 [loggers.py:248] Engine 000: Avg prompt throughput: 274.2 tokens/s, Avg generation throughput: 261.8 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40118 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:51354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:26 [loggers.py:248] Engine 000: Avg prompt throughput: 97.1 tokens/s, Avg generation throughput: 219.6 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:36 [loggers.py:248] Engine 000: Avg prompt throughput: 94.5 tokens/s, Avg generation throughput: 180.2 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43262 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:46 [loggers.py:248] Engine 000: Avg prompt throughput: 106.3 tokens/s, Avg generation throughput: 143.5 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:42760 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43278 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:40500 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:52126 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:45070 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:45084 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:52:56 [loggers.py:248] Engine 000: Avg prompt throughput: 230.9 tokens/s, Avg generation throughput: 187.0 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:45084 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58040 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:58034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:45100 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:06 [loggers.py:248] Engine 000: Avg prompt throughput: 122.7 tokens/s, Avg generation throughput: 248.2 tokens/s, Running: 8 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:16 [loggers.py:248] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 103.4 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.4%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:26 [loggers.py:248] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 16.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:36 [loggers.py:248] Engine 000: Avg prompt throughput: 321.3 tokens/s, Avg generation throughput: 125.4 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.2%, Prefix cache hit rate: 7.4%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:46 [loggers.py:248] Engine 000: Avg prompt throughput: 1139.4 tokens/s, Avg generation throughput: 526.7 tokens/s, Running: 28 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.7%, Prefix cache hit rate: 26.7%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:53:56 [loggers.py:248] Engine 000: Avg prompt throughput: 1121.3 tokens/s, Avg generation throughput: 805.2 tokens/s, Running: 32 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.0%, Prefix cache hit rate: 39.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41640 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55104 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:06 [loggers.py:248] Engine 000: Avg prompt throughput: 612.6 tokens/s, Avg generation throughput: 864.9 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 13.3%, Prefix cache hit rate: 43.9%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55104 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:16 [loggers.py:248] Engine 000: Avg prompt throughput: 620.8 tokens/s, Avg generation throughput: 856.1 tokens/s, Running: 35 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.3%, Prefix cache hit rate: 47.8%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41876 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:26 [loggers.py:248] Engine 000: Avg prompt throughput: 922.2 tokens/s, Avg generation throughput: 816.9 tokens/s, Running: 39 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.7%, Prefix cache hit rate: 42.6%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37690 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37716 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37738 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55104 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:36 [loggers.py:248] Engine 000: Avg prompt throughput: 981.1 tokens/s, Avg generation throughput: 967.6 tokens/s, Running: 42 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.1%, Prefix cache hit rate: 38.2%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:46 [loggers.py:248] Engine 000: Avg prompt throughput: 481.1 tokens/s, Avg generation throughput: 802.9 tokens/s, Running: 32 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.7%, Prefix cache hit rate: 36.4%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37716 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56422 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37738 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:54:56 [loggers.py:248] Engine 000: Avg prompt throughput: 966.1 tokens/s, Avg generation throughput: 930.3 tokens/s, Running: 50 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.5%, Prefix cache hit rate: 33.2%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37716 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41664 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41854 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56422 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44350 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37738 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44414 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:06 [loggers.py:248] Engine 000: Avg prompt throughput: 447.5 tokens/s, Avg generation throughput: 1072.4 tokens/s, Running: 36 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.9%, Prefix cache hit rate: 31.9%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37738 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56392 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44388 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37730 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37674 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37716 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:16 [loggers.py:248] Engine 000: Avg prompt throughput: 809.3 tokens/s, Avg generation throughput: 813.6 tokens/s, Running: 33 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.0%, Prefix cache hit rate: 29.8%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53936 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54002 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53962 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53982 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:26 [loggers.py:248] Engine 000: Avg prompt throughput: 991.2 tokens/s, Avg generation throughput: 638.6 tokens/s, Running: 25 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.7%, Prefix cache hit rate: 27.5%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:36 [loggers.py:248] Engine 000: Avg prompt throughput: 492.4 tokens/s, Avg generation throughput: 661.6 tokens/s, Running: 26 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.9%, Prefix cache hit rate: 26.5%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41910 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53538 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53558 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53558 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53578 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53578 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53580 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:46 [loggers.py:248] Engine 000: Avg prompt throughput: 946.8 tokens/s, Avg generation throughput: 797.0 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.5%, Prefix cache hit rate: 25.0%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:55:56 [loggers.py:248] Engine 000: Avg prompt throughput: 737.3 tokens/s, Avg generation throughput: 746.0 tokens/s, Running: 22 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.3%, Prefix cache hit rate: 23.8%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44416 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53578 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53538 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53580 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53580 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:56:06 [loggers.py:248] Engine 000: Avg prompt throughput: 743.4 tokens/s, Avg generation throughput: 694.3 tokens/s, Running: 36 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.7%, Prefix cache hit rate: 22.7%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53998 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54010 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44390 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41894 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:56:16 [loggers.py:248] Engine 000: Avg prompt throughput: 606.7 tokens/s, Avg generation throughput: 723.4 tokens/s, Running: 28 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.0%, Prefix cache hit rate: 21.8%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37718 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41630 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:54024 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53974 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53580 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44398 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:44372 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53578 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53900 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56408 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56436 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:56452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41648 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60364 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43272 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43288 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43296 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43298 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43300 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43302 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43288 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53548 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41892 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43272 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO 12-09 18:56:26 [loggers.py:248] Engine 000: Avg prompt throughput: 976.4 tokens/s, Avg generation throughput: 808.8 tokens/s, Running: 35 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.9%, Prefix cache hit rate: 20.6%
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53976 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55086 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43302 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:43300 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53950 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:41912 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:55106 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53564 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=3950)[0;0m INFO: 127.0.0.1:53892 - "POST /v1/completions HTTP/1.1" 200 OK
|