1033 lines
114 KiB
Plaintext
1033 lines
114 KiB
Plaintext
|
WARNING 12-09 19:04:10 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13.
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:10 [api_server.py:1772] vLLM API server version 0.12.0
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:10 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64}
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:16 [model.py:637] Resolved architecture: Qwen3ForCausalLM
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:16 [model.py:1750] Using max model len 32768
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:17 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:23 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>}, 'local_cache_dir': None}
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:24 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:39463 backend=nccl
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:24 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:24 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic...
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:24 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION']
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 8.25it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 3.27it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:01<00:00, 2.62it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.33it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.61it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:27 [default_loader.py:308] Loading weights took 1.61 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:27 [gpu_model_runner.py:3549] Model loading took 15.3388 GiB memory and 2.767054 seconds
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:37 [backends.py:655] Using cache directory: /root/.cache/vllm/torch_compile_cache/930c5e773b/rank_0_0/backbone for vLLM's torch.compile
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:37 [backends.py:715] Dynamo bytecode transform time: 8.89 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:38 [backends.py:257] Cache the graph for dynamic shape for later use
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:43 [backends.py:288] Compiling a graph for dynamic shape takes 5.81 s
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:47 [monitor.py:34] torch.compile takes 14.70 s in total
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:48 [gpu_worker.py:359] Available KV cache memory: 12.32 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:48 [kv_cache_utils.py:1286] GPU KV cache size: 80,704 tokens
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:48 [kv_cache_utils.py:1291] Maximum concurrency for 32,768 tokens per request: 2.46x
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m 2025-12-09 19:04:48,493 - INFO - autotuner.py:256 - flashinfer.jit: [Autotuner]: Autotuning process starts ...
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m 2025-12-09 19:04:48,503 - INFO - autotuner.py:262 - flashinfer.jit: [Autotuner]: Autotuning process ends
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:00<00:00, 26.39it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:00<00:00, 26.68it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:00<00:00, 26.67it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:00<00:00, 26.77it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:00<00:00, 26.67it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:00<00:00, 26.36it/s]
|
||
|
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:00<00:00, 26.44it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m
|
||
|
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:01, 5.90it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:00<00:00, 16.60it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:00<00:00, 21.37it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:00<00:00, 23.85it/s]
|
||
|
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:00<00:00, 21.12it/s]
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:50 [gpu_model_runner.py:4466] Graph capturing finished in 2 secs, took -0.24 GiB
|
||
|
|
[0;36m(EngineCore_DP0 pid=14297)[0;0m INFO 12-09 19:04:50 [core.py:254] init engine (profile, create kv cache, warmup model) took 22.52 seconds
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [api_server.py:1520] Supported tasks: ['generate']
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m WARNING 12-09 19:04:52 [model.py:1576] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [serving_responses.py:194] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [serving_completion.py:73] Using default completion sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [serving_chat.py:133] Using default chat sampling params from model: {'temperature': 0.6, 'top_k': 20, 'top_p': 0.95}
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [api_server.py:1847] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:38] Available routes are:
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /openapi.json, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /docs, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /redoc, Methods: GET, HEAD
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /health, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /load, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /pause, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /resume, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /is_paused, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /tokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /detokenize, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/models, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /version, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /ping, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /ping, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /invocations, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /metrics, Methods: GET
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /classify, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/score, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:04:52 [launcher.py:46] Route: /pooling, Methods: POST
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: Started server process [14033]
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: Waiting for application startup.
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: Application startup complete.
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:58726 - "GET /v1/models HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:05:13 [loggers.py:236] Engine 000: Avg prompt throughput: 2.4 tokens/s, Avg generation throughput: 13.8 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38562 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:05:23 [loggers.py:236] Engine 000: Avg prompt throughput: 114.3 tokens/s, Avg generation throughput: 131.6 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.9%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45836 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:05:33 [loggers.py:236] Engine 000: Avg prompt throughput: 246.5 tokens/s, Avg generation throughput: 224.3 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:05:43 [loggers.py:236] Engine 000: Avg prompt throughput: 244.2 tokens/s, Avg generation throughput: 151.0 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:05:53 [loggers.py:236] Engine 000: Avg prompt throughput: 249.4 tokens/s, Avg generation throughput: 190.5 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.7%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:59908 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52208 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52220 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:03 [loggers.py:236] Engine 000: Avg prompt throughput: 342.7 tokens/s, Avg generation throughput: 217.6 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52208 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52220 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52220 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:13 [loggers.py:236] Engine 000: Avg prompt throughput: 284.7 tokens/s, Avg generation throughput: 202.6 tokens/s, Running: 6 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.1%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49714 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49714 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38568 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:23 [loggers.py:236] Engine 000: Avg prompt throughput: 371.6 tokens/s, Avg generation throughput: 246.6 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:33 [loggers.py:236] Engine 000: Avg prompt throughput: 247.8 tokens/s, Avg generation throughput: 191.7 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.6%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38554 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37676 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51820 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:43 [loggers.py:236] Engine 000: Avg prompt throughput: 169.1 tokens/s, Avg generation throughput: 363.8 tokens/s, Running: 10 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.9%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:49710 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51796 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:06:53 [loggers.py:236] Engine 000: Avg prompt throughput: 339.2 tokens/s, Avg generation throughput: 297.4 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:03 [loggers.py:236] Engine 000: Avg prompt throughput: 38.5 tokens/s, Avg generation throughput: 190.6 tokens/s, Running: 4 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:13 [loggers.py:236] Engine 000: Avg prompt throughput: 201.2 tokens/s, Avg generation throughput: 208.1 tokens/s, Running: 2 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45914 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:45924 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51810 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:23 [loggers.py:236] Engine 000: Avg prompt throughput: 246.3 tokens/s, Avg generation throughput: 229.7 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.9%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:46728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:37662 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:33 [loggers.py:236] Engine 000: Avg prompt throughput: 99.5 tokens/s, Avg generation throughput: 303.8 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:43692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52192 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:52234 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:43 [loggers.py:236] Engine 000: Avg prompt throughput: 93.6 tokens/s, Avg generation throughput: 92.6 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47666 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47696 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47698 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47696 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47708 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:07:53 [loggers.py:236] Engine 000: Avg prompt throughput: 120.4 tokens/s, Avg generation throughput: 166.3 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.3%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47696 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47702 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47708 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47708 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:50298 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:03 [loggers.py:236] Engine 000: Avg prompt throughput: 161.8 tokens/s, Avg generation throughput: 219.8 tokens/s, Running: 5 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47708 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:51794 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47696 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:36650 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:36658 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:36658 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:50298 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:50298 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:47704 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:13 [loggers.py:236] Engine 000: Avg prompt throughput: 263.8 tokens/s, Avg generation throughput: 243.9 tokens/s, Running: 7 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:23 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 152.3 tokens/s, Running: 1 reqs, Waiting: 0 reqs, GPU KV cache usage: 1.2%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:33 [loggers.py:236] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 3.5 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:43 [loggers.py:236] Engine 000: Avg prompt throughput: 428.1 tokens/s, Avg generation throughput: 223.1 tokens/s, Running: 9 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 9.6%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:08:53 [loggers.py:236] Engine 000: Avg prompt throughput: 1264.1 tokens/s, Avg generation throughput: 747.0 tokens/s, Running: 24 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.9%, Prefix cache hit rate: 29.5%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41628 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:03 [loggers.py:236] Engine 000: Avg prompt throughput: 926.0 tokens/s, Avg generation throughput: 889.0 tokens/s, Running: 21 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.6%, Prefix cache hit rate: 39.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38706 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38706 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38706 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:13 [loggers.py:236] Engine 000: Avg prompt throughput: 672.5 tokens/s, Avg generation throughput: 1021.0 tokens/s, Running: 20 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.9%, Prefix cache hit rate: 44.4%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:23 [loggers.py:236] Engine 000: Avg prompt throughput: 669.6 tokens/s, Avg generation throughput: 724.3 tokens/s, Running: 23 reqs, Waiting: 0 reqs, GPU KV cache usage: 11.4%, Prefix cache hit rate: 47.2%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:33 [loggers.py:236] Engine 000: Avg prompt throughput: 945.1 tokens/s, Avg generation throughput: 1039.1 tokens/s, Running: 24 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.3%, Prefix cache hit rate: 42.1%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41654 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:43 [loggers.py:236] Engine 000: Avg prompt throughput: 934.3 tokens/s, Avg generation throughput: 825.9 tokens/s, Running: 19 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.6%, Prefix cache hit rate: 38.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:09:53 [loggers.py:236] Engine 000: Avg prompt throughput: 503.5 tokens/s, Avg generation throughput: 782.5 tokens/s, Running: 22 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.7%, Prefix cache hit rate: 36.2%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:03 [loggers.py:236] Engine 000: Avg prompt throughput: 966.8 tokens/s, Avg generation throughput: 1130.1 tokens/s, Running: 31 reqs, Waiting: 0 reqs, GPU KV cache usage: 12.0%, Prefix cache hit rate: 33.0%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42012 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:13 [loggers.py:236] Engine 000: Avg prompt throughput: 590.6 tokens/s, Avg generation throughput: 986.8 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.8%, Prefix cache hit rate: 31.4%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35404 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41668 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38722 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:23 [loggers.py:236] Engine 000: Avg prompt throughput: 1023.1 tokens/s, Avg generation throughput: 774.1 tokens/s, Running: 20 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.5%, Prefix cache hit rate: 28.8%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41656 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41612 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41680 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35432 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:33 [loggers.py:236] Engine 000: Avg prompt throughput: 729.7 tokens/s, Avg generation throughput: 658.0 tokens/s, Running: 22 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.0%, Prefix cache hit rate: 27.3%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:43 [loggers.py:236] Engine 000: Avg prompt throughput: 704.5 tokens/s, Avg generation throughput: 698.1 tokens/s, Running: 23 reqs, Waiting: 0 reqs, GPU KV cache usage: 8.5%, Prefix cache hit rate: 25.9%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:10:53 [loggers.py:236] Engine 000: Avg prompt throughput: 723.4 tokens/s, Avg generation throughput: 928.3 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.3%, Prefix cache hit rate: 24.8%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41618 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:11:03 [loggers.py:236] Engine 000: Avg prompt throughput: 902.9 tokens/s, Avg generation throughput: 677.2 tokens/s, Running: 17 reqs, Waiting: 0 reqs, GPU KV cache usage: 7.2%, Prefix cache hit rate: 23.4%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:11:13 [loggers.py:236] Engine 000: Avg prompt throughput: 759.7 tokens/s, Avg generation throughput: 760.6 tokens/s, Running: 22 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.1%, Prefix cache hit rate: 22.3%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:11:23 [loggers.py:236] Engine 000: Avg prompt throughput: 495.1 tokens/s, Avg generation throughput: 860.4 tokens/s, Running: 21 reqs, Waiting: 0 reqs, GPU KV cache usage: 10.0%, Prefix cache hit rate: 21.7%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33782 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41590 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35378 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35374 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41576 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39224 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33790 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35356 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41566 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35366 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41600 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38682 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39226 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35420 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35368 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO 12-09 19:11:33 [loggers.py:236] Engine 000: Avg prompt throughput: 1132.6 tokens/s, Avg generation throughput: 737.1 tokens/s, Running: 18 reqs, Waiting: 0 reqs, GPU KV cache usage: 6.0%, Prefix cache hit rate: 20.3%
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38692 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39198 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:42028 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:39210 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:41638 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56788 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35394 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:56798 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35384 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:38728 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:33768 - "POST /v1/completions HTTP/1.1" 200 OK
|
||
|
|
[0;36m(APIServer pid=14033)[0;0m INFO: 127.0.0.1:35786 - "POST /v1/completions HTTP/1.1" 200 OK
|