commit a04b9ef9c5807c3937429e93fd2166aebb347518 Author: SZCJW <792430652@qq.com> Date: Thu Mar 26 20:50:00 2026 +0800 x diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 0000000..5778bd2 --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,98 @@ +name: build-and-publish + +on: + workflow_dispatch: + inputs: + tag: + description: "Image tag to publish (e.g. latest)" + required: true + default: "latest" + +env: + IMAGE_REPO: kyuz0/vllm-therock-gfx1201 + DOCKER_BUILDKIT: "1" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Put Docker on /mnt + run: | + set -eux + echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json + sudo systemctl stop docker + sudo rm -rf /var/lib/docker || true + sudo mkdir -p /mnt/docker + sudo systemctl start docker + docker info | grep "Docker Root Dir" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Free disk space + shell: bash + run: | + set -euxo pipefail + echo "Disk BEFORE:"; df -h + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true + sudo rm -rf /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go || true + docker system prune -af || true + docker builder prune -af || true + sudo apt-get clean + sudo rm -rf /var/lib/apt/lists/* + sudo rm -rf /opt/hostedtoolcache + echo "Disk AFTER:"; df -h + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: BuildKit GC config (8GB cap) + run: | + cat > /tmp/buildkitd.toml <<'EOF' + [worker.oci] + gc = true + gckeepstorage = 8000 # MB + EOF + + - name: Set up Buildx (with GC) + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: --config /tmp/buildkitd.toml + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: docker.io/${{ env.IMAGE_REPO }} + tags: | + type=raw,value=${{ github.event.inputs.tag }} + type=sha + type=raw,value={{date 'YYYYMMDD-HHmmss'}} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + provenance: false + sbom: false + no-cache: true + + - name: Prune buildx cache + if: always() + run: | + docker buildx prune -af --verbose --min-free-space 4gb || true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed8ebf5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..10b731c --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml +# 基于编辑器的 HTTP 客户端请求 +/httpRequests/ diff --git a/.idea/amd-r9700-vllm-toolboxes.iml b/.idea/amd-r9700-vllm-toolboxes.iml new file mode 100644 index 0000000..2a15c21 --- /dev/null +++ b/.idea/amd-r9700-vllm-toolboxes.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..c677c95 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..eb2d36c --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..0159947 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8853520 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,164 @@ +FROM registry.fedoraproject.org/fedora:43 + +# 1. System Base & Build Tools +# Added 'gperftools-libs' for tcmalloc (fixes double-free) +RUN dnf -y install --setopt=install_weak_deps=False --nodocs \ + python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \ + gcc gcc-c++ binutils make ffmpeg-free \ + cmake ninja-build aria2c tar xz vim nano \ + libdrm-devel zlib-devel openssl-devel jq \ + numactl-devel gperftools-libs dialog procps-ng \ + && dnf clean all && rm -rf /var/cache/dnf/* + +# 2. Install "TheRock" ROCm SDK (Tarball Method) +WORKDIR /tmp +ARG ROCM_MAJOR_VER=7 +ARG GFX=gfx120X-all +RUN set -euo pipefail; \ + BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \ + PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \ + KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \ + | tr '<' '\n' \ + | grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \ + | sort -V | tail -n1)"; \ + echo "Downloading Latest Tarball: ${KEY}"; \ + aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \ + mkdir -p /opt/rocm; \ + tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \ + rm therock.tar.gz + +# 3. Configure Global ROCm Environment +# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash +RUN export ROCM_PATH=/opt/rocm && \ + BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \ + printf '%s\n' \ + "export ROCM_PATH=/opt/rocm" \ + "export HIP_PLATFORM=amd" \ + "export HIP_PATH=/opt/rocm" \ + "export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \ + "export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \ + "export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \ + "export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \ + "export ROCBLAS_USE_HIPBLASLT=1" \ + "export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \ + "export VLLM_TARGET_DEVICE=rocm" \ + "export HIP_FORCE_DEV_KERNARG=1" \ + "export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \ + "export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \ + > /etc/profile.d/rocm-sdk.sh && \ + chmod 0644 /etc/profile.d/rocm-sdk.sh + +# 4. Python Venv Setup +RUN /usr/bin/python3.13 -m venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=/opt/venv/bin:$PATH +ENV PIP_NO_CACHE_DIR=1 +RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh +RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0" + +# 5. Install PyTorch (TheRock Nightly) +RUN python -m pip install \ + --index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \ + --pre torch torchaudio torchvision + +# Flash-Attention +WORKDIR /opt +ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE" + +RUN git clone https://github.com/ROCm/flash-attention.git &&\ + cd flash-attention &&\ + git checkout main_perf &&\ + python setup.py install && \ + cd /opt && rm -rf /opt/flash-attention + +# 6. Clone vLLM +RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm +WORKDIR /opt/vllm + +# --- PATCHING --- +# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers), +# vLLM falls back to CPU. We patch it to force ROCm detection. +RUN echo "import sys, re" > patch_vllm.py && \ + echo "from pathlib import Path" >> patch_vllm.py && \ + # Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks + echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \ + echo "txt = p.read_text()" >> patch_vllm.py && \ + echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \ + echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \ + echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \ + echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \ + echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \ + echo "p.write_text(txt)" >> patch_vllm.py && \ + # Patch 2: rocm.py - Mock amdsmi and force device name + echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \ + echo "txt = p.read_text()" >> patch_vllm.py && \ + echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \ + echo "txt = header + txt" >> patch_vllm.py && \ + echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \ + echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \ + echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \ + echo "p.write_text(txt)" >> patch_vllm.py && \ + echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \ + python patch_vllm.py + +# 7. Build vLLM (Wheel Method) with CLANG Host Compiler +RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11 +ENV ROCM_HOME="/opt/rocm" +ENV HIP_PATH="/opt/rocm" +ENV VLLM_TARGET_DEVICE="rocm" +ENV PYTORCH_ROCM_ARCH="gfx1201" +ENV HIP_ARCHITECTURES="gfx1201" +ENV AMDGPU_TARGETS="gfx1201" +ENV MAX_JOBS="4" + +# --- FIX FOR SEGFAULT --- +# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC. +# This aligns the ABI of the compiled vLLM extensions with PyTorch. +ENV CC="/opt/rocm/llvm/bin/clang" +ENV CXX="/opt/rocm/llvm/bin/clang++" + +RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \ + echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \ + export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \ + export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \ + python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \ + python -m pip install /tmp/dist/*.whl + +# --- bitsandbytes (ROCm) --- +WORKDIR /opt +RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git +WORKDIR /opt/bitsandbytes + +# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile) +ENV HIP_PLATFORM="amd" +ENV CMAKE_PREFIX_PATH="/opt/rocm" + +# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++) +RUN cmake -S . \ + -DGPU_TARGETS="gfx1201" \ + -DBNB_ROCM_ARCH="gfx1201" \ + -DCOMPUTE_BACKEND=hip \ + -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \ + -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ + && \ + make -j$(nproc) && \ + python -m pip install --no-cache-dir . --no-build-isolation --no-deps + +# 8. Final Cleanup & Runtime +WORKDIR /opt +RUN chmod -R a+rwX /opt && \ + find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \ + find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \ + rm -rf /root/.cache/pip || true && \ + dnf clean all && rm -rf /var/cache/dnf/* + +COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh +COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh +COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh +COPY scripts/start_vllm.py /usr/local/bin/start-vllm +COPY benchmarks/max_context_results.json /opt/max_context_results.json +COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py +RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json +RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh + +CMD ["/bin/bash"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..9f3a1fd --- /dev/null +++ b/README.md @@ -0,0 +1,209 @@ +# AMD Radeon 9700 AI PRO (gfx1201) — vLLM Toolbox/Container + +An **fedora-based** Docker/Podman container that is **Toolbx-compatible** (usable as a Fedora toolbox) for serving LLMs with **vLLM** on **AMD Radeon R9700 (gfx1201)**. Built on the TheRock nightly builds for ROCM. + +![Demo](demo.gif) + +--- + +## Table of Contents + +* [Tested Models (Benchmarks)](#tested-models-benchmarks) +* [1) Toolbx vs Docker/Podman](#1-toolbx-vs-dockerpodman) +* [2) Quickstart — Fedora Toolbx](#2-quickstart--fedora-toolbx) +* [3) Quickstart — Ubuntu (Distrobox)](#3-quickstart--ubuntu-distrobox) +* [4) Testing the API](#4-testing-the-api) +* [5) Use a Web UI for Chatting](#5-use-a-web-ui-for-chatting) + + +## Tested Models (Benchmarks) + +View full benchmarks at: [https://kyuz0.github.io/amd-r9700-vllm-toolboxes/](https://kyuz0.github.io/amd-r9700-vllm-toolboxes/) + +*Run benchmarks now include a comparison between the default Triton backend and the optional ROCm attention backend.* + + +**Table Key:** Cell values represent `Max Context Length (GPU Memory Utilization)`. + +| Model | TP | 1 Req | 4 Reqs | 8 Reqs | 16 Reqs | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **`meta-llama/Meta-Llama-3.1-8B-Instruct`** | 1 | 127k (0.98) | 127k (0.98) | 127k (0.98) | 127k (0.98) | +| | 2 | 105k (0.98) | 105k (0.98) | 105k (0.98) | 105k (0.98) | +| **`openai/gpt-oss-20b`** | 1 | 131k (0.98) | 131k (0.98) | 131k (0.98) | 131k (0.98) | +| | 2 | 131k (0.95) | 131k (0.95) | 131k (0.95) | 131k (0.95) | +| **`RedHatAI/Qwen3-14B-FP8-dynamic`** | 1 | 41k (0.98) | 41k (0.98) | 41k (0.98) | 41k (0.98) | +| | 2 | 41k (0.95) | 41k (0.95) | 41k (0.95) | 41k (0.95) | +| **`cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit`** | 1 | 151k (0.98) | 151k (0.98) | 151k (0.98) | 151k (0.98) | +| | 2 | 262k (0.98) | 262k (0.98) | 262k (0.98) | 262k (0.98) | +| **`cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit`** | 2 | 156k (0.98) | 156k (0.98) | 156k (0.98) | 156k (0.98) | +| **`RedHatAI/gemma-3-12b-it-FP8-dynamic`** | 1 | 45k (0.98) | 45k (0.98) | 45k (0.98) | 45k (0.98) | +| | 2 | 126k (0.98) | 126k (0.98) | 121k (0.95) | 121k (0.95) | +| **`RedHatAI/gemma-3-27b-it-FP8-dynamic`** | 2 | 60k (0.98) | 60k (0.98) | 60k (0.98) | 60k (0.98) | + +### Advanced Tuning + +See [TUNING.md](TUNING.md) for a guide on how to enable undervolting and raise the power limit on AMD R9700 cards on Linux to improve performance and efficiency. + +### 🆕 Update: Comparison of Attention Backends (Triton vs ROCm) + +*Added Support for ROCm Native Attention Backend* + +I have added the ability to switch between the default **Triton** backend and the experimental **ROCm** native backend for attention operations. This provides you with more flexibility to optimize for stability or throughput depending on your specific model and workload. + +| Backend | Stability | Throughput | Compatibility | +| :--- | :--- | :--- | :--- | +| **Triton** (Default) | ✅ **High** | 🔸 Good | Works with all tested models | +| **ROCm** | ⚠️ **Experimental** | 🚀 **Highest** | May fail with complex architectures | + +**Key Differences:** +- **Triton**: The safe choice. It uses the Triton compiler to generate kernels and is the standard for vLLM on AMD. +- **ROCm**: Uses composable kernel based attention. In my benchmarks, this often yields higher throughput (tokens/sec) but can be less stable, leading to crashes or "invalid graph" errors on some newer models. + +**How to Use:** +1. **Easy Mode**: Select the backend in the `start-vllm` wizard (Item 5 in the menu). +2. **Manual Mode**: Export the following environment variables before running `vllm serve`: + ```bash + export VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 + export VLLM_USE_TRITON_FLASH_ATTN=0 + ``` + + +--- + +## 1) Toolbx vs Docker/Podman + +The `kyuz0/vllm-therock-gfx1201:latest` image can be used both as:  + +* **Fedora Toolbx (recommended for development):** Toolbx shares your **HOME** and user, so models/configs live on the host. Great for iterating quickly while keeping the host clean. +* **Docker/Podman (recommended for deployment/perf):** Use for running vLLM as a service (host networking, IPC tuning, etc.). Always **mount a host directory** for model weights so they stay outside the container. + +--- + +## 2) Quickstart — Fedora Toolbx + +Create a toolbox that exposes the GPU and relaxes seccomp to avoid ROCm syscall issues: + +```bash +toolbox create vllm-r9700 \ + --image docker.io/kyuz0/vllm-therock-gfx1201:latest \ + -- --device /dev/dri --device /dev/kfd \ + --group-add video --group-add render --security-opt seccomp=unconfined +``` + +Enter it: + +```bash +toolbox enter vllm-r9700 +``` + +**Model storage:** Models are downloaded to `~/.cache/huggingface` by default. This directory is shared with the host if you created the toolbox correctly, so downloads persist. + +### Serving a Model (Easiest Way) + +The toolbox includes a TUI wizard called **`start-vllm`** which includes pre-configured models and handles launch flags. It also allows you to select the experimental **ROCm attention backend**. This is the easiest way to get started. + +```bash +# if your weights live on disk instead of on HuggingFace, point the +# launcher at the directory that contains the model folders. the +# script will look for a subdirectory matching the repo ID and use it +# when launching. +export LOCAL_MODEL_DIR=/workspace/models +start-vllm + +# you can also just run the CLI yourself and pass the path directly: +vllm serve /workspace/models/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit ... +``` + +> **Cache note:** vLLM writes compiled kernels to `~/.cache/vllm/`. + +--- + +### 离线/本地模式 + +如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。 +脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**， +不会尝试下载任何内容。适用于没有外网或受限环境的部署。 + +```bash +export LOCAL_MODEL_DIR=/workspace/models +start-vllm # 本地模式，如果模型缺失则失败 +``` + +### 离线/本地模式 + +如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。 +脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**， +不会尝试下载任何内容。适用于没有外网或受限环境的部署。 + +```bash +export LOCAL_MODEL_DIR=/workspace/models +start-vllm # 本地模式，如果模型缺失则失败 +``` + +## 3) Quickstart — Ubuntu (Distrobox) + +Ubuntu’s toolbox package still breaks GPU access, so use Distrobox instead: + +```bash +distrobox create -n vllm-r9700 \ + --image docker.io/kyuz0/vllm-therock-gfx1201:latest \ + --additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined" + +distrobox enter vllm-r9700 +``` + +> **Verification:** Run `rocm-smi` to check GPU status. + +### Serving a Model +Same as above, you can use the **`start-vllm`** wizard to launch models easily. + +```bash +start-vllm +``` + +--- + +## 4) Testing the API + +Once the server is up, hit the OpenAI‑compatible endpoint: + +```bash +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello! Test the performance."}]}' +``` + +You should receive a JSON response with a `choices[0].message.content` reply. + +If you don't want to bother specifying the model name, you can run this which will query the currently deployed model: + +```bash +MODEL=$(curl -s http://localhost:8000/v1/models | jq -r '.data[0].id') curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\":[{\"role\":\"user\",\"content\":\"Hello! Test the performance.\"}] + }" +``` + +--- + +## 5) Use a Web UI for Chatting + +If vLLM is on a remote server, expose port 8000 via SSH port forwarding: + +```bash +ssh -L 0.0.0.0:8000:localhost:8000 +``` + +Then, you can start HuggingFace ChatUI like this (on your host): + +```bash +docker run -p 3000:3000 \ + --add-host=host.docker.internal:host-gateway \ + -e OPENAI_BASE_URL=http://host.docker.internal:8000/v1 \ + -e OPENAI_API_KEY=dummy \ + -v chat-ui-data:/data \ + ghcr.io/huggingface/chat-ui-db +``` + diff --git a/TUNING.md b/TUNING.md new file mode 100644 index 0000000..185fb04 --- /dev/null +++ b/TUNING.md @@ -0,0 +1,108 @@ +# AMD Radeon 9700 AI PRO (Navi 48) Tuning Guide + +This guide show you how to enable undervolting and raise the power limit on the Radeon PRO R9700 on Fedora 43. + +## 1. Prerequisites (Enable Overclocking) + +The `ppfeaturemask` must be set to unlock voltage control. + +```bash +sudo grubby --update-kernel=ALL --args="amdgpu.ppfeaturemask=0xffffffff" +``` + +**Reboot your system after running this.** + +## 2. Identify Your GPU + +You need the **PCI Bus ID** of the target card (e.g., `07:00.0`). + +Run this command to list your AMD GPUs: + +```bash +lspci -nn | grep "VGA" | grep "AMD" +``` + +*Example output:* +`07:00.0 VGA compatible controller...` +*(In this example, your ID is `0000:07:00.0`)* + +## 3. Tuning Script + +Save this script as `tune_r9700.sh`. + +**Important:** Edit the `PCI_ID` variable at the top to match the ID you found in Step 2. + +```bash +#!/bin/bash + +# --- CONFIGURATION --- +# Replace this with your specific PCI ID from 'lspci' +# Format must be 0000:XX:XX.X +PCI_ID="0000:07:00.0" +# --------------------- + +# 1. Robustly Find the Card Name (e.g., card1) directly from the PCI Bus +# We look inside the PCI device's 'drm' folder for a folder starting with 'card' followed only by numbers. +if [ ! -d "/sys/bus/pci/devices/$PCI_ID/drm" ]; then + echo "Error: PCI Device $PCI_ID not found or has no DRM driver attached." + exit 1 +fi + +# This finds 'card1' but ignores 'card1-DP-6' +CARD_NAME=$(ls "/sys/bus/pci/devices/$PCI_ID/drm" | grep -E '^card[0-9]+$' | head -n 1) + +if [ -z "$CARD_NAME" ]; then + echo "Error: Could not determine card name for $PCI_ID" + exit 1 +fi + +# Construct the clean path (e.g., /sys/class/drm/card1) +CARD_PATH="/sys/class/drm/$CARD_NAME" + +echo "Tuning GPU: $CARD_NAME (at $CARD_PATH)..." + +# 2. Force Manual Performance Level (Required for UV) +# We use 'tee' without pipe to ensure exact errors are caught, but your 'echo | sudo tee' method is fine. +echo "manual" | sudo tee "$CARD_PATH/device/power_dpm_force_performance_level" > /dev/null +if [ $? -ne 0 ]; then echo "Failed to set Manual mode. Check permissions/path."; exit 1; fi + +# 3. Apply -75mV Undervolt +echo "vo -75" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null +echo "c" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null +echo "Applied Undervolt (-75mV)" + +# 4. Set Power Limit to 315W +# Find the hwmon directory strictly inside the device +HWMON_DIR=$(find "$CARD_PATH/device/hwmon/" -maxdepth 1 -name "hwmon*" | head -n 1) +if [ -n "$HWMON_DIR" ]; then + echo "315000000" | sudo tee "$HWMON_DIR/power1_cap" > /dev/null + echo "Applied Power Limit (315W)" +else + echo "Error: Could not find hwmon directory for power limit." +fi +``` + +### Usage + +```bash +chmod +x tune_r9700.sh +sudo ./tune_r9700.sh +``` + +## 4\. Verification + +Run these commands to confirm settings are active. + +**Check Undervolt:** + +```bash +# Look for: OD_VDDGFX_OFFSET: -75mV +cat /sys/class/drm/card0/device/pp_od_clk_voltage +``` + +**Check Power Limit:** + +```bash +# Should be ~315 W +cat /sys/class/drm/card0/device/hwmon/hwmon*/power1_cap +``` \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json new file mode 100644 index 0000000..7a531dc --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 14:59:00 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ce1f54c0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.72 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.76 \nPeak output token throughput (tok/s): 344.00 \nPeak concurrent requests: 16.00 \nTotal token throughput (tok/s): 391.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.14 \nMedian TTFT (ms): 87.12 \nP99 TTFT (ms): 220.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.41 \nMedian TPOT (ms): 41.23 \nP99 TPOT (ms): 46.01 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.43 \nMedian ITL (ms): 40.25 \nP99 ITL (ms): 86.38 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..2621b7e --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:02:31 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-959ca2a7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 668 \nFailed requests: 52 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 180.01 \nTotal input tokens: 137901 \nTotal generated tokens: 128399 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 713.31 \nPeak output token throughput (tok/s): 1215.00 \nPeak concurrent requests: 72.00 \nTotal token throughput (tok/s): 1479.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 123.92 \nMedian TTFT (ms): 95.36 \nP99 TTFT (ms): 403.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.89 \nMedian TPOT (ms): 55.38 \nP99 TPOT (ms): 442.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.97 \nMedian ITL (ms): 49.28 \nP99 ITL (ms): 187.29 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..f9a58ad --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,16815 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 14:58:16 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=58748) INFO 12-19 14:58:16 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=58748) INFO 12-19 14:58:16 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=58748) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=58748) INFO 12-19 14:58:20 [model.py:514] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=58748) INFO 12-19 14:58:20 [model.py:1636] Using max model len 32768 +(APIServer pid=58748) INFO 12-19 14:58:20 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=58913) INFO 12-19 14:58:24 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=58913) INFO 12-19 14:58:24 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:48181 backend=nccl +(EngineCore_DP0 pid=58913) INFO 12-19 14:58:24 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=58913) INFO 12-19 14:58:24 [gpu_model_runner.py:3562] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=58913) INFO 12-19 14:58:25 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(EngineCore_DP0 pid=58913) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, 'debug_dump_path': None, 'cache_dir': '/home/kyuz0/.cache/vllm/torch_compile_cache/824e5832da', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': '/home/kyuz0/.cache/vllm/torch_compile_cache/824e5832da/rank_0_0/backbone'}, +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [dump_input.py:79] Dumping scheduler output for model execution: SchedulerOutput(scheduled_new_reqs=[NewRequestData(req_id=cmpl-bench-959ca2a7-668-0,prompt_token_ids_len=597,mm_features=[],sampling_params=SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.0, top_p=1.0, top_k=0, min_p=0.0, seed=None, stop=[], stop_token_ids=[151643], bad_words=[], include_stop_str_in_output=False, ignore_eos=False, max_tokens=11, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, truncate_prompt_tokens=None, structured_outputs=None, extra_args=None),block_ids=([2457, 311, 4955, 4403, 501, 1834, 3391, 1462, 679, 1386, 2343, 978, 2420, 2359, 2291, 1348, 611, 4569, 4357, 1443, 2301, 4246, 3135, 4341, 5307, 1176, 1744, 4249, 2188, 1792, 1334, 4950, 562, 5082, 127, 4989, 4837, 4530],),num_computed_tokens=0,lora_request=None,prompt_embeds_shape=None)], scheduled_cached_reqs=CachedRequestData(req_ids=['cmpl-bench-959ca2a7-518-0', 'cmpl-bench-959ca2a7-553-0', 'cmpl-bench-959ca2a7-573-0', 'cmpl-bench-959ca2a7-582-0', 'cmpl-bench-959ca2a7-586-0', 'cmpl-bench-959ca2a7-596-0', 'cmpl-bench-959ca2a7-602-0', 'cmpl-bench-959ca2a7-606-0', 'cmpl-bench-959ca2a7-607-0', 'cmpl-bench-959ca2a7-608-0', 'cmpl-bench-959ca2a7-609-0', 'cmpl-bench-959ca2a7-613-0', 'cmpl-bench-959ca2a7-617-0', 'cmpl-bench-959ca2a7-619-0', 'cmpl-bench-959ca2a7-622-0', 'cmpl-bench-959ca2a7-624-0', 'cmpl-bench-959ca2a7-625-0', 'cmpl-bench-959ca2a7-626-0', 'cmpl-bench-959ca2a7-627-0', 'cmpl-bench-959ca2a7-628-0', 'cmpl-bench-959ca2a7-630-0', 'cmpl-bench-959ca2a7-631-0', 'cmpl-bench-959ca2a7-632-0', 'cmpl-bench-959ca2a7-633-0', 'cmpl-bench-959ca2a7-634-0', 'cmpl-bench-959ca2a7-635-0', 'cmpl-bench-959ca2a7-636-0', 'cmpl-bench-959ca2a7-637-0', 'cmpl-bench-959ca2a7-638-0', 'cmpl-bench-959ca2a7-639-0', 'cmpl-bench-959ca2a7-640-0', 'cmpl-bench-959ca2a7-641-0', 'cmpl-bench-959ca2a7-642-0', 'cmpl-bench-959ca2a7-643-0', 'cmpl-bench-959ca2a7-648-0', 'cmpl-bench-959ca2a7-649-0', 'cmpl-bench-959ca2a7-650-0', 'cmpl-bench-959ca2a7-655-0', 'cmpl-bench-959ca2a7-656-0', 'cmpl-bench-959ca2a7-658-0', 'cmpl-bench-959ca2a7-659-0', 'cmpl-bench-959ca2a7-660-0', 'cmpl-bench-959ca2a7-661-0', 'cmpl-bench-959ca2a7-662-0', 'cmpl-bench-959ca2a7-663-0', 'cmpl-bench-959ca2a7-664-0', 'cmpl-bench-959ca2a7-665-0', 'cmpl-bench-959ca2a7-666-0', 'cmpl-bench-959ca2a7-667-0'], resumed_req_ids=[], new_token_ids=[], all_token_ids={}, new_block_ids=[null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, [[585]], null, null, null, null, null, [[1203]], null, null, null, null, null, [[4226]], null, null, null, null, [[5315]], null, null, null, null, null, null, null, null], num_computed_tokens=[743, 623, 1340, 663, 450, 395, 767, 317, 374, 461, 386, 385, 486, 541, 231, 211, 856, 191, 183, 185, 204, 191, 242, 224, 166, 168, 173, 157, 951, 352, 143, 121, 230, 115, 193, 80, 129, 58, 175, 52, 32, 104, 25, 25, 781, 533, 22, 622, 583], num_output_tokens=[738, 606, 483, 434, 426, 353, 318, 312, 311, 307, 305, 285, 277, 238, 227, 198, 186, 181, 175, 173, 172, 171, 171, 167, 159, 143, 142, 142, 137, 133, 124, 110, 89, 86, 65, 65, 61, 49, 48, 45, 25, 23, 12, 12, 10, 9, 8, 4, 3]), num_scheduled_tokens={cmpl-bench-959ca2a7-613-0: 1, cmpl-bench-959ca2a7-602-0: 1, cmpl-bench-959ca2a7-622-0: 1, cmpl-bench-959ca2a7-518-0: 1, cmpl-bench-959ca2a7-617-0: 1, cmpl-bench-959ca2a7-655-0: 1, cmpl-bench-959ca2a7-643-0: 1, cmpl-bench-959ca2a7-663-0: 1, cmpl-bench-959ca2a7-596-0: 1, cmpl-bench-959ca2a7-637-0: 1, cmpl-bench-959ca2a7-631-0: 1, cmpl-bench-959ca2a7-638-0: 1, cmpl-bench-959ca2a7-573-0: 1, cmpl-bench-959ca2a7-642-0: 1, cmpl-bench-959ca2a7-619-0: 1, cmpl-bench-959ca2a7-607-0: 1, cmpl-bench-959ca2a7-658-0: 1, cmpl-bench-959ca2a7-635-0: 1, cmpl-bench-959ca2a7-632-0: 1, cmpl-bench-959ca2a7-634-0: 1, cmpl-bench-959ca2a7-633-0: 1, cmpl-bench-959ca2a7-662-0: 1, cmpl-bench-959ca2a7-659-0: 1, cmpl-bench-959ca2a7-630-0: 1, cmpl-bench-959ca2a7-624-0: 1, cmpl-bench-959ca2a7-628-0: 1, cmpl-bench-959ca2a7-667-0: 1, cmpl-bench-959ca2a7-606-0: 1, cmpl-bench-959ca2a7-626-0: 1, cmpl-bench-959ca2a7-553-0: 1, cmpl-bench-959ca2a7-608-0: 1, cmpl-bench-959ca2a7-661-0: 1, cmpl-bench-959ca2a7-664-0: 1, cmpl-bench-959ca2a7-627-0: 1, cmpl-bench-959ca2a7-641-0: 1, cmpl-bench-959ca2a7-649-0: 1, cmpl-bench-959ca2a7-656-0: 1, cmpl-bench-959ca2a7-636-0: 1, cmpl-bench-959ca2a7-625-0: 1, cmpl-bench-959ca2a7-639-0: 1, cmpl-bench-959ca2a7-648-0: 1, cmpl-bench-959ca2a7-666-0: 1, cmpl-bench-959ca2a7-609-0: 1, cmpl-bench-959ca2a7-665-0: 1, cmpl-bench-959ca2a7-668-0: 597, cmpl-bench-959ca2a7-660-0: 1, cmpl-bench-959ca2a7-640-0: 1, cmpl-bench-959ca2a7-586-0: 1, cmpl-bench-959ca2a7-650-0: 1, cmpl-bench-959ca2a7-582-0: 1}, total_num_scheduled_tokens=646, scheduled_spec_decode_tokens={}, scheduled_encoder_inputs={}, num_common_prefix_blocks=[0], finished_req_ids=[], free_encoder_mm_hashes=[], preempted_req_ids=[], pending_structured_output_tokens=false, kv_connector_metadata=null, ec_connector_metadata=null) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [dump_input.py:81] Dumping scheduler stats: SchedulerStats(num_running_reqs=50, num_waiting_reqs=0, step_counter=0, current_wave=0, kv_cache_usage=0.181803197626504, prefix_cache_stats=PrefixCacheStats(reset=False, requests=1, queries=597, hits=0, preempted_requests=0, preempted_queries=0, preempted_hits=0), connector_prefix_cache_stats=None, kv_cache_eviction_events=[], spec_decoding_stats=None, kv_connector_stats=None, waiting_lora_adapters={}, running_lora_adapters={}, cudagraph_stats=None) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] EngineCore encountered a fatal error. +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] Traceback (most recent call last): +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 859, in run_engine_core +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] engine_core.run_busy_loop() +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 886, in run_busy_loop +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] self._process_engine_step() +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 919, in _process_engine_step +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] outputs, model_executed = self.step_fn() +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 353, in step +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] model_output = self.model_executor.sample_tokens(grammar_output) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/uniproc_executor.py", line 110, in sample_tokens +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] return self.collective_rpc( +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] "sample_tokens", +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ...<2 lines>... +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] single_value=True, +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] result = run_method(self.driver_worker, method, args, kwargs) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/serial_utils.py", line 461, in run_method +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 572, in sample_tokens +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] return self.model_runner.sample_tokens(grammar_output) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 3281, in sample_tokens +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ) = self._bookkeeping_sync( +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] scheduler_output, +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ...<4 lines>... +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] spec_decode_metadata, +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 2625, in _bookkeeping_sync +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] valid_sampled_token_ids = self._to_list(sampled_token_ids) +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5492, in _to_list +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] self.transfer_event.synchronize() +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] torch.AcceleratorError: HIP error: an illegal memory access was encountered +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] For debugging consider passing AMD_SERIALIZE_KERNEL=3 +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. +(EngineCore_DP0 pid=58913) ERROR 12-19 15:05:29 [core.py:868] +(EngineCore_DP0 pid=58913) Process EngineCore_DP0: +(EngineCore_DP0 pid=58913) Traceback (most recent call last): +(EngineCore_DP0 pid=58913) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap +(EngineCore_DP0 pid=58913) self.run() +(EngineCore_DP0 pid=58913) ~~~~~~~~^^ +(EngineCore_DP0 pid=58913) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run +(EngineCore_DP0 pid=58913) self._target(*self._args, **self._kwargs) +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +(EngineCore_DP0 pid=58913) raise e +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 859, in run_engine_core +(EngineCore_DP0 pid=58913) engine_core.run_busy_loop() +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 886, in run_busy_loop +(EngineCore_DP0 pid=58913) self._process_engine_step() +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 919, in _process_engine_step +(EngineCore_DP0 pid=58913) outputs, model_executed = self.step_fn() +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 353, in step +(EngineCore_DP0 pid=58913) model_output = self.model_executor.sample_tokens(grammar_output) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/uniproc_executor.py", line 110, in sample_tokens +(EngineCore_DP0 pid=58913) return self.collective_rpc( +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=58913) "sample_tokens", +(EngineCore_DP0 pid=58913) ^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ...<2 lines>... +(EngineCore_DP0 pid=58913) single_value=True, +(EngineCore_DP0 pid=58913) ^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ) +(EngineCore_DP0 pid=58913) ^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/uniproc_executor.py", line 75, in collective_rpc +(EngineCore_DP0 pid=58913) result = run_method(self.driver_worker, method, args, kwargs) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/serial_utils.py", line 461, in run_method +(EngineCore_DP0 pid=58913) return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(EngineCore_DP0 pid=58913) return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 572, in sample_tokens +(EngineCore_DP0 pid=58913) return self.model_runner.sample_tokens(grammar_output) +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(EngineCore_DP0 pid=58913) return func(*args, **kwargs) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 3281, in sample_tokens +(EngineCore_DP0 pid=58913) ) = self._bookkeeping_sync( +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=58913) scheduler_output, +(EngineCore_DP0 pid=58913) ^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ...<4 lines>... +(EngineCore_DP0 pid=58913) spec_decode_metadata, +(EngineCore_DP0 pid=58913) ^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=58913) ) +(EngineCore_DP0 pid=58913) ^ +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 2625, in _bookkeeping_sync +(EngineCore_DP0 pid=58913) valid_sampled_token_ids = self._to_list(sampled_token_ids) +(EngineCore_DP0 pid=58913) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5492, in _to_list +(EngineCore_DP0 pid=58913) self.transfer_event.synchronize() +(EngineCore_DP0 pid=58913) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=58913) torch.AcceleratorError: HIP error: an illegal memory access was encountered +(EngineCore_DP0 pid=58913) Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +(EngineCore_DP0 pid=58913) HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +(EngineCore_DP0 pid=58913) For debugging consider passing AMD_SERIALIZE_KERNEL=3 +(EngineCore_DP0 pid=58913) Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. +(EngineCore_DP0 pid=58913) +[rank0]:[W1219 15:05:29.875861010 ProcessGroupNCCL.cpp:1553] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] AsyncLLM output_handler failed. +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [async_llm.py:538] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +hipModuleUnload failed: + error: an illegal memory access was encountered +hipModuleUnload failed: + error: an illegal memory access was encountered +hipModuleUnload failed: + error: an illegal memory access was encountered +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:29 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] yield 0, item +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise output +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=58748) ERROR 12-19 15:05:30 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=58748) INFO: 127.0.0.1:50762 - "POST /v1/completions HTTP/1.1" 500 Internal Server Error +(APIServer pid=58748) INFO: 127.0.0.1:52514 - "POST /v1/completions HTTP/1.1" 500 Internal Server Error +(APIServer pid=58748) INFO: 127.0.0.1:52528 - "POST /v1/completions HTTP/1.1" 500 Internal Server Error +(APIServer pid=58748) INFO: Shutting down +(APIServer pid=58748) INFO: Waiting for application shutdown. +(APIServer pid=58748) INFO: Application shutdown complete. +(APIServer pid=58748) INFO: Finished server process [58748] diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json new file mode 100644 index 0000000..e791f92 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json @@ -0,0 +1,7 @@ +{ + "elapsed_time": 518.4320176600013, + "num_requests": 1000, + "total_num_tokens": 741334, + "requests_per_second": 1.9288932124863885, + "tokens_per_second": 1429.9541207853842 +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_server.log new file mode 100644 index 0000000..d024178 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_server.log @@ -0,0 +1,432 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 17:09:03 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=75372) INFO 12-19 17:09:03 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=75372) INFO 12-19 17:09:03 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32000, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=75372) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=75372) INFO 12-19 17:09:08 [model.py:514] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=75372) INFO 12-19 17:09:08 [model.py:1636] Using max model len 32000 +(APIServer pid=75372) INFO 12-19 17:09:08 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=75534) INFO 12-19 17:09:12 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32000, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=75534) WARNING 12-19 17:09:12 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl +INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl +INFO 12-19 17:09:16 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 17:09:16 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-19 17:09:16 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=75616) INFO 12-19 17:09:17 [gpu_model_runner.py:3562] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(Worker_TP1 pid=75617) INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=75616) INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=75616) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00... +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits, +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception. +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits, +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits, +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] EngineCore failed to start. +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] Traceback (most recent call last): +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] super().__init__( +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] vllm_config +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] return aggregate(get_response()) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] raise RuntimeError( +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ...<2 lines>... +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ) +(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause +(EngineCore_DP0 pid=75534) Process EngineCore_DP0: +(EngineCore_DP0 pid=75534) Traceback (most recent call last): +(EngineCore_DP0 pid=75534) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap +(EngineCore_DP0 pid=75534) self.run() +(EngineCore_DP0 pid=75534) ~~~~~~~~^^ +(EngineCore_DP0 pid=75534) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run +(EngineCore_DP0 pid=75534) self._target(*self._args, **self._kwargs) +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +(EngineCore_DP0 pid=75534) raise e +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=75534) engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=75534) super().__init__( +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75534) vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=75534) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ) +(EngineCore_DP0 pid=75534) ^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=75534) num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75534) vllm_config +(EngineCore_DP0 pid=75534) ^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) ) +(EngineCore_DP0 pid=75534) ^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=75534) self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=75534) self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=75534) return aggregate(get_response()) +(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=75534) raise RuntimeError( +(EngineCore_DP0 pid=75534) ...<2 lines>... +(EngineCore_DP0 pid=75534) ) +(EngineCore_DP0 pid=75534) RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause +(Worker_TP1 pid=75617) INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker +(Worker_TP0 pid=75616) INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker +(APIServer pid=75372) Traceback (most recent call last): +(APIServer pid=75372) File "/opt/venv/bin/vllm", line 7, in +(APIServer pid=75372) sys.exit(main()) +(APIServer pid=75372) ~~~~^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main +(APIServer pid=75372) args.dispatch_function(args) +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd +(APIServer pid=75372) uvloop.run(run_server(args)) +(APIServer pid=75372) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run +(APIServer pid=75372) return __asyncio.run( +(APIServer pid=75372) ~~~~~~~~~~~~~^ +(APIServer pid=75372) wrapper(), +(APIServer pid=75372) ^^^^^^^^^^ +(APIServer pid=75372) ...<2 lines>... +(APIServer pid=75372) **run_kwargs +(APIServer pid=75372) ^^^^^^^^^^^^ +(APIServer pid=75372) ) +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run +(APIServer pid=75372) return runner.run(main) +(APIServer pid=75372) ~~~~~~~~~~^^^^^^ +(APIServer pid=75372) File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run +(APIServer pid=75372) return self._loop.run_until_complete(task) +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=75372) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper +(APIServer pid=75372) return await main +(APIServer pid=75372) ^^^^^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server +(APIServer pid=75372) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker +(APIServer pid=75372) async with build_async_engine_client( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=75372) args, +(APIServer pid=75372) ^^^^^ +(APIServer pid=75372) client_config=client_config, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ) as engine_client: +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=75372) return await anext(self.gen) +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client +(APIServer pid=75372) async with build_async_engine_client_from_engine_args( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=75372) engine_args, +(APIServer pid=75372) ^^^^^^^^^^^^ +(APIServer pid=75372) ...<2 lines>... +(APIServer pid=75372) client_config=client_config, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ) as engine: +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=75372) return await anext(self.gen) +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args +(APIServer pid=75372) async_llm = AsyncLLM.from_vllm_config( +(APIServer pid=75372) vllm_config=vllm_config, +(APIServer pid=75372) ...<6 lines>... +(APIServer pid=75372) client_index=client_index, +(APIServer pid=75372) ) +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config +(APIServer pid=75372) return cls( +(APIServer pid=75372) vllm_config=vllm_config, +(APIServer pid=75372) ...<9 lines>... +(APIServer pid=75372) client_index=client_index, +(APIServer pid=75372) ) +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__ +(APIServer pid=75372) self.engine_core = EngineCoreClient.make_async_mp_client( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=75372) vllm_config=vllm_config, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ...<4 lines>... +(APIServer pid=75372) client_index=client_index, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ) +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client +(APIServer pid=75372) return AsyncMPClient(*client_args) +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__ +(APIServer pid=75372) super().__init__( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~^ +(APIServer pid=75372) asyncio_mode=True, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ...<3 lines>... +(APIServer pid=75372) client_addresses=client_addresses, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ) +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 477, in __init__ +(APIServer pid=75372) with launch_core_engines(vllm_config, executor_class, log_stats) as ( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) File "/usr/lib64/python3.13/contextlib.py", line 148, in __exit__ +(APIServer pid=75372) next(self.gen) +(APIServer pid=75372) ~~~~^^^^^^^^^^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 903, in launch_core_engines +(APIServer pid=75372) wait_for_engine_startup( +(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=75372) handshake_socket, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ...<5 lines>... +(APIServer pid=75372) coordinator.proc if coordinator else None, +(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=75372) ) +(APIServer pid=75372) ^ +(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup +(APIServer pid=75372) raise RuntimeError( +(APIServer pid=75372) ...<3 lines>... +(APIServer pid=75372) ) +(APIServer pid=75372) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_throughput.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_throughput.json new file mode 100644 index 0000000..59fbb6e --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_throughput.json @@ -0,0 +1,7 @@ +{ + "elapsed_time": 470.41839592500037, + "num_requests": 1000, + "total_num_tokens": 741334, + "requests_per_second": 2.1257672077931065, + "tokens_per_second": 1575.9035072220947 +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps1.0_latency.json new file mode 100644 index 0000000..aa1d86d --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:31:43 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-242c7681-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.10 \nTotal input tokens: 40429 \nTotal generated tokens: 38744 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 191.71 \nPeak output token throughput (tok/s): 354.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 391.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 127.38 \nMedian TTFT (ms): 107.28 \nP99 TTFT (ms): 286.14 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.15 \nMedian TPOT (ms): 45.01 \nP99 TPOT (ms): 56.92 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.71 \nMedian ITL (ms): 42.72 \nP99 ITL (ms): 114.80 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..66a918c --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:35:15 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-8a860b01-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 224.44 \nTotal input tokens: 158086 \nTotal generated tokens: 153768 \nRequest throughput (req/s): 3.21 \nOutput token throughput (tok/s): 685.11 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 111.00 \nTotal token throughput (tok/s): 1389.47 \n---------------Time to First Token----------------\nMean TTFT (ms): 3686.81 \nMedian TTFT (ms): 3310.63 \nP99 TTFT (ms): 10615.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 75.05 \nMedian TPOT (ms): 75.32 \nP99 TPOT (ms): 105.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 74.21 \nMedian ITL (ms): 62.11 \nP99 ITL (ms): 246.46 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..963249d --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log @@ -0,0 +1,1031 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 15:30:50 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=60902) INFO 12-19 15:30:50 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=60902) INFO 12-19 15:30:50 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 9900, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=60902) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=60902) INFO 12-19 15:30:54 [model.py:514] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=60902) INFO 12-19 15:30:54 [model.py:1636] Using max model len 9900 +(APIServer pid=60902) INFO 12-19 15:30:55 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=61066) INFO 12-19 15:30:59 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/gemma-3-12b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-12b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=9900, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-12b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=61066) INFO 12-19 15:31:00 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:36665 backend=nccl +(EngineCore_DP0 pid=61066) INFO 12-19 15:31:00 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=61066) Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(EngineCore_DP0 pid=61066) INFO 12-19 15:31:06 [gpu_model_runner.py:3562] Starting to load model RedHatAI/gemma-3-12b-it-FP8-dynamic... +(EngineCore_DP0 pid=61066) WARNING 12-19 15:31:06 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=61066) INFO 12-19 15:31:06 [layer.py:537] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(EngineCore_DP0 pid=61066) WARNING 12-19 15:31:06 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(EngineCore_DP0 pid=61066) INFO 12-19 15:31:06 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(EngineCore_DP0 pid=61066) WARNING 12-19 15:31:06 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(EngineCore_DP0 pid=61066) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b978cded-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.92 \nTotal input tokens: 40429 \nTotal generated tokens: 38727 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 196.66 \nPeak output token throughput (tok/s): 393.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 401.97 \n---------------Time to First Token----------------\nMean TTFT (ms): 232.41 \nMedian TTFT (ms): 152.06 \nP99 TTFT (ms): 582.73 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 39.99 \nMedian TPOT (ms): 38.66 \nP99 TPOT (ms): 75.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.94 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 310.76 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json new file mode 100644 index 0000000..06f2684 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 18:07:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3d8d3c55-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 299.13 \nTotal input tokens: 158086 \nTotal generated tokens: 152394 \nRequest throughput (req/s): 2.41 \nOutput token throughput (tok/s): 509.46 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 294.00 \nTotal token throughput (tok/s): 1037.94 \n---------------Time to First Token----------------\nMean TTFT (ms): 37309.58 \nMedian TTFT (ms): 42371.37 \nP99 TTFT (ms): 86881.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 113.18 \nMedian TPOT (ms): 113.38 \nP99 TPOT (ms): 240.19 \n---------------Inter-token Latency----------------\nMean ITL (ms): 109.51 \nMedian ITL (ms): 58.62 \nP99 ITL (ms): 472.97 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log new file mode 100644 index 0000000..04ff57b --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log @@ -0,0 +1,1053 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 18:02:43 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=80667) INFO 12-19 18:02:43 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=80667) INFO 12-19 18:02:43 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 9900, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=80667) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=80667) INFO 12-19 18:02:47 [model.py:514] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=80667) INFO 12-19 18:02:47 [model.py:1636] Using max model len 9900 +(APIServer pid=80667) INFO 12-19 18:02:48 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=80832) INFO 12-19 18:02:52 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/gemma-3-12b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-12b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=9900, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-12b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=80832) WARNING 12-19 18:02:52 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 18:02:57 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:37123 backend=nccl +INFO 12-19 18:02:57 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:37123 backend=nccl +INFO 12-19 18:02:57 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 18:02:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-19 18:02:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(Worker_TP0 pid=80914) INFO 12-19 18:03:03 [gpu_model_runner.py:3562] Starting to load model RedHatAI/gemma-3-12b-it-FP8-dynamic... +(Worker_TP0 pid=80914) WARNING 12-19 18:03:04 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=80914) INFO 12-19 18:03:04 [layer.py:537] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP0 pid=80914) WARNING 12-19 18:03:04 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP1 pid=80915) WARNING 12-19 18:03:04 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=80915) INFO 12-19 18:03:04 [layer.py:537] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP1 pid=80915) WARNING 12-19 18:03:04 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP0 pid=80914) INFO 12-19 18:03:04 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=80914) WARNING 12-19 18:03:04 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP1 pid=80915) INFO 12-19 18:03:04 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP1 pid=80915) WARNING 12-19 18:03:04 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=80914) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0a333e52-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 45 \nFailed requests: 135 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 180.00 \nTotal input tokens: 10566 \nTotal generated tokens: 7279 \nRequest throughput (req/s): 0.25 \nOutput token throughput (tok/s): 40.44 \nPeak output token throughput (tok/s): 234.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 99.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 235.33 \nMedian TTFT (ms): 121.50 \nP99 TTFT (ms): 734.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 55.13 \nMedian TPOT (ms): 54.64 \nP99 TPOT (ms): 75.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.59 \nMedian ITL (ms): 50.79 \nP99 ITL (ms): 355.12 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json new file mode 100644 index 0000000..c0c69e0 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": false, + "raw_output": "WARNING 12-19 17:40:29 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-660ac132-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log new file mode 100644 index 0000000..b16c6e3 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log @@ -0,0 +1,2570 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 17:35:32 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=77686) INFO 12-19 17:35:32 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=77686) INFO 12-19 17:35:32 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-27b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-27b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 28900, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.94, 'max_num_seqs': 32} +(APIServer pid=77686) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=77686) INFO 12-19 17:35:36 [model.py:514] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=77686) INFO 12-19 17:35:36 [model.py:1636] Using max model len 28900 +(APIServer pid=77686) INFO 12-19 17:35:36 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=77851) INFO 12-19 17:35:41 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/gemma-3-27b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-27b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=28900, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-27b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=77851) WARNING 12-19 17:35:41 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 17:35:46 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:33879 backend=nccl +INFO 12-19 17:35:46 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:33879 backend=nccl +INFO 12-19 17:35:46 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 17:35:46 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-19 17:35:46 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(Worker_TP0 pid=77933) INFO 12-19 17:35:52 [gpu_model_runner.py:3562] Starting to load model RedHatAI/gemma-3-27b-it-FP8-dynamic... +(Worker_TP0 pid=77933) WARNING 12-19 17:35:53 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=77933) INFO 12-19 17:35:53 [layer.py:537] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP0 pid=77933) WARNING 12-19 17:35:53 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP1 pid=77934) WARNING 12-19 17:35:53 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=77934) INFO 12-19 17:35:53 [layer.py:537] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP1 pid=77934) WARNING 12-19 17:35:53 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP1 pid=77934) INFO 12-19 17:35:53 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=77933) INFO 12-19 17:35:53 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP1 pid=77934) WARNING 12-19 17:35:53 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=77933) WARNING 12-19 17:35:53 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=77933) Loading safetensors checkpoint shards: 0% Completed | 0/6 [00:00... +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] spec_decode_metadata, +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 2625, in _bookkeeping_sync +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] valid_sampled_token_ids = self._to_list(sampled_token_ids) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5492, in _to_list +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] self.transfer_event.synchronize() +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] torch.AcceleratorError: HIP error: an illegal memory access was encountered +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] For debugging consider passing AMD_SERIALIZE_KERNEL=3 +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 572, in sample_tokens +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] return self.model_runner.sample_tokens(grammar_output) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 3281, in sample_tokens +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ) = self._bookkeeping_sync( +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] scheduler_output, +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] spec_decode_metadata, +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 2625, in _bookkeeping_sync +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] valid_sampled_token_ids = self._to_list(sampled_token_ids) +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5492, in _to_list +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] self.transfer_event.synchronize() +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] torch.AcceleratorError: HIP error: an illegal memory access was encountered +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] For debugging consider passing AMD_SERIALIZE_KERNEL=3 +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] +(Worker_TP1 pid=77934) ERROR 12-19 17:38:16 [multiproc_executor.py:826] +[rank1]:[E1219 17:38:16.877431190 ProcessGroupNCCL.cpp:2093] [PG ID 2 PG GUID 3 Rank 1] Process group watchdog thread terminated with exception: HIP error: an illegal memory access was encountered +Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing AMD_SERIALIZE_KERNEL=3 +Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. + +Exception raised from getDevice at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/hip/impl/HIPGuardImplMasqueradingAsCUDA.h:75 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x9d (0x7f01713fafdd in /opt/venv/lib64/python3.13/site-packages/torch/lib/libc10.so) +frame #1: + 0x12568 (0x7f0171486568 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libc10_hip.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x922 (0x7f0174a7ccc2 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libtorch_hip.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0x105 (0x7f0174a7f1b5 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libtorch_hip.so) +frame #4: + 0x4e3e4 (0x7f01dce603e4 in /lib64/libstdc++.so.6) +frame #5: + 0x72464 (0x7f01dd0fb464 in /lib64/libc.so.6) +frame #6: + 0xf55ac (0x7f01dd17e5ac in /lib64/libc.so.6) + +terminate called after throwing an instance of 'c10::DistBackendError' + what(): [PG ID 2 PG GUID 3 Rank 1] Process group watchdog thread terminated with exception: HIP error: an illegal memory access was encountered +Search for `hipErrorIllegalAddress' in https://rocm.docs.amd.com/projects/HIP/en/latest/index.html for more information. +HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing AMD_SERIALIZE_KERNEL=3 +Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions. + +Exception raised from getDevice at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/hip/impl/HIPGuardImplMasqueradingAsCUDA.h:75 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x9d (0x7f01713fafdd in /opt/venv/lib64/python3.13/site-packages/torch/lib/libc10.so) +frame #1: + 0x12568 (0x7f0171486568 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libc10_hip.so) +frame #2: c10d::ProcessGroupNCCL::Watchdog::runLoop() + 0x922 (0x7f0174a7ccc2 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libtorch_hip.so) +frame #3: c10d::ProcessGroupNCCL::Watchdog::run() + 0x105 (0x7f0174a7f1b5 in /opt/venv/lib64/python3.13/site-packages/torch/lib/libtorch_hip.so) +frame #4: + 0x4e3e4 (0x7f01dce603e4 in /lib64/libstdc++.so.6) +frame #5: + 0x72464 (0x7f01dd0fb464 in /lib64/libc.so.6) +frame #6: + 0xf55ac (0x7f01dd17e5ac in /lib64/libc.so.6) + +Exception raised from run at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/torch/csrc/distributed/c10d/ProcessGroupNCCL.cpp:2099 (most recent call first): +frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x9d (0x7f01713fafdd in /opt/venv/lib64/python3.13/site-packages/torch/lib/libc10.so) +frame #1: + 0xb51c0c (0x7f0172129c0c in /opt/venv/lib64/python3.13/site-packages/torch/lib/libtorch_hip.so) +frame #2: + 0x4e3e4 (0x7f01dce603e4 in /lib64/libstdc++.so.6) +frame #3: + 0x72464 (0x7f01dd0fb464 in /lib64/libc.so.6) +frame #4: + 0xf55ac (0x7f01dd17e5ac in /lib64/libc.so.6) + +(APIServer pid=77686) INFO: 127.0.0.1:60172 - "POST /v1/completions HTTP/1.1" 200 OK +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:16 [multiproc_executor.py:233] Worker proc VllmWorker-1 died unexpectedly, shutting down executor. +(Worker_TP0 pid=77933) INFO 12-19 17:38:16 [multiproc_executor.py:711] Parent process exited, terminating worker +(APIServer pid=77686) INFO: 127.0.0.1:59118 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=77686) INFO: 127.0.0.1:59124 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=77686) INFO: 127.0.0.1:59126 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=77686) INFO 12-19 17:38:18 [loggers.py:248] Engine 000: Avg prompt throughput: 172.0 tokens/s, Avg generation throughput: 150.8 tokens/s, Running: 11 reqs, Waiting: 0 reqs, GPU KV cache usage: 9.1%, Prefix cache hit rate: 0.0%, MM cache hit rate: 0.0% +(APIServer pid=77686) INFO: 127.0.0.1:59134 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=77686) INFO: 127.0.0.1:59138 - "POST /v1/completions HTTP/1.1" 200 OK +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [dump_input.py:72] Dumping input data for V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/gemma-3-27b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-27b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=28900, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-27b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None}, +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [dump_input.py:79] Dumping scheduler output for model execution: SchedulerOutput(scheduled_new_reqs=[NewRequestData(req_id=cmpl-bench-0a333e52-45-0,prompt_token_ids_len=282,mm_features=[],sampling_params=SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.0, top_p=1.0, top_k=0, min_p=0.0, seed=None, stop=[], stop_token_ids=[106], bad_words=[], include_stop_str_in_output=False, ignore_eos=False, max_tokens=493, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, truncate_prompt_tokens=None, structured_outputs=None, extra_args=None),block_ids=([7974, 7975, 7976, 7977, 7978, 7979, 7980, 7981, 7982, 7983, 7984, 7985, 7986, 7987, 7988, 7989, 7990, 7991], [7992, 7993, 7994, 7995, 7996, 7997, 7998, 7999, 8000, 8001, 8002, 8003, 8004, 8005, 8006, 8007, 8008, 8009], [8010, 8011, 8012, 8013, 8014, 8015, 8016, 8017, 8018, 8019, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8027], [8028, 8029, 8030, 8031, 8032, 8033, 8034, 8035, 8036, 8037, 8038, 8039, 8040, 8041, 8042, 8043, 8044, 8045], [8046, 8047, 8048, 8049, 8050, 8051, 8052, 8053, 8054, 8055, 8056, 8057, 8058, 8059, 8060, 8061, 8062, 8063], [8064, 8065, 8066, 8067, 8068, 8069, 8070, 8071, 8072, 8073, 8074, 8075, 8076, 8077, 8078, 8079, 8080, 8081], [8082, 8083, 8084, 8085, 8086, 8087, 8088, 8089, 8090, 8091, 8092, 8093, 8094, 8095, 8096, 8097, 8098, 8099]),num_computed_tokens=0,lora_request=None,prompt_embeds_shape=None)], scheduled_cached_reqs=CachedRequestData(req_ids=['cmpl-bench-0a333e52-5-0', 'cmpl-bench-0a333e52-20-0', 'cmpl-bench-0a333e52-21-0', 'cmpl-bench-0a333e52-26-0', 'cmpl-bench-0a333e52-34-0', 'cmpl-bench-0a333e52-35-0', 'cmpl-bench-0a333e52-36-0', 'cmpl-bench-0a333e52-39-0', 'cmpl-bench-0a333e52-40-0', 'cmpl-bench-0a333e52-43-0', 'cmpl-bench-0a333e52-44-0'], resumed_req_ids=[], new_token_ids=[], all_token_ids={}, new_block_ids=[null, null, null, null, null, null, null, null, null, null, null], num_computed_tokens=[877, 1009, 430, 726, 538, 213, 838, 87, 79, 47, 27], num_output_tokens=[849, 401, 372, 323, 184, 177, 159, 72, 63, 36, 21]), num_scheduled_tokens={cmpl-bench-0a333e52-40-0: 1, cmpl-bench-0a333e52-43-0: 1, cmpl-bench-0a333e52-44-0: 1, cmpl-bench-0a333e52-45-0: 282, cmpl-bench-0a333e52-39-0: 1, cmpl-bench-0a333e52-20-0: 1, cmpl-bench-0a333e52-5-0: 1, cmpl-bench-0a333e52-26-0: 1, cmpl-bench-0a333e52-21-0: 1, cmpl-bench-0a333e52-34-0: 1, cmpl-bench-0a333e52-36-0: 1, cmpl-bench-0a333e52-35-0: 1}, total_num_scheduled_tokens=293, scheduled_spec_decode_tokens={}, scheduled_encoder_inputs={}, num_common_prefix_blocks=[0, 0, 0, 0, 0, 0, 0], finished_req_ids=[], free_encoder_mm_hashes=[], preempted_req_ids=[], pending_structured_output_tokens=false, kv_connector_metadata=null, ec_connector_metadata=null) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [dump_input.py:81] Dumping scheduler stats: SchedulerStats(num_running_reqs=12, num_waiting_reqs=0, step_counter=0, current_wave=0, kv_cache_usage=0.09605539236256821, prefix_cache_stats=PrefixCacheStats(reset=False, requests=1, queries=282, hits=0, preempted_requests=0, preempted_queries=0, preempted_hits=0), connector_prefix_cache_stats=None, kv_cache_eviction_events=[], spec_decoding_stats=None, kv_connector_stats=None, waiting_lora_adapters={}, running_lora_adapters={}, cudagraph_stats=None) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] EngineCore encountered a fatal error. +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] Traceback (most recent call last): +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 859, in run_engine_core +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] engine_core.run_busy_loop() +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 886, in run_busy_loop +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] self._process_engine_step() +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 919, in _process_engine_step +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] outputs, model_executed = self.step_fn() +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 353, in step +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] model_output = self.model_executor.sample_tokens(grammar_output) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 271, in sample_tokens +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] return self.collective_rpc( +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] "sample_tokens", +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ...<4 lines>... +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] kv_output_aggregator=self.kv_output_aggregator, +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] return aggregate(get_response()) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 338, in get_response +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] status, result = mq.dequeue( +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] timeout=dequeue_timeout, cancel=shutdown_event +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/distributed/device_communicators/shm_broadcast.py", line 616, in dequeue +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] with self.acquire_read(timeout, cancel, indefinite) as buf: +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/usr/lib64/python3.13/contextlib.py", line 141, in __enter__ +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] return next(self.gen) +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] File "/opt/venv/lib64/python3.13/site-packages/vllm/distributed/device_communicators/shm_broadcast.py", line 531, in acquire_read +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] raise RuntimeError("cancelled") +(EngineCore_DP0 pid=77851) ERROR 12-19 17:38:20 [core.py:868] RuntimeError: cancelled +(EngineCore_DP0 pid=77851) Process EngineCore_DP0: +(EngineCore_DP0 pid=77851) Traceback (most recent call last): +(EngineCore_DP0 pid=77851) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap +(EngineCore_DP0 pid=77851) self.run() +(EngineCore_DP0 pid=77851) ~~~~~~~~^^ +(EngineCore_DP0 pid=77851) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run +(EngineCore_DP0 pid=77851) self._target(*self._args, **self._kwargs) +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +(EngineCore_DP0 pid=77851) raise e +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 859, in run_engine_core +(EngineCore_DP0 pid=77851) engine_core.run_busy_loop() +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 886, in run_busy_loop +(EngineCore_DP0 pid=77851) self._process_engine_step() +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~~~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 919, in _process_engine_step +(EngineCore_DP0 pid=77851) outputs, model_executed = self.step_fn() +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 353, in step +(EngineCore_DP0 pid=77851) model_output = self.model_executor.sample_tokens(grammar_output) +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 271, in sample_tokens +(EngineCore_DP0 pid=77851) return self.collective_rpc( +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=77851) "sample_tokens", +(EngineCore_DP0 pid=77851) ^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ...<4 lines>... +(EngineCore_DP0 pid=77851) kv_output_aggregator=self.kv_output_aggregator, +(EngineCore_DP0 pid=77851) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ) +(EngineCore_DP0 pid=77851) ^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=77851) return aggregate(get_response()) +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 338, in get_response +(EngineCore_DP0 pid=77851) status, result = mq.dequeue( +(EngineCore_DP0 pid=77851) ~~~~~~~~~~^ +(EngineCore_DP0 pid=77851) timeout=dequeue_timeout, cancel=shutdown_event +(EngineCore_DP0 pid=77851) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) ) +(EngineCore_DP0 pid=77851) ^ +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/distributed/device_communicators/shm_broadcast.py", line 616, in dequeue +(EngineCore_DP0 pid=77851) with self.acquire_read(timeout, cancel, indefinite) as buf: +(EngineCore_DP0 pid=77851) ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=77851) File "/usr/lib64/python3.13/contextlib.py", line 141, in __enter__ +(EngineCore_DP0 pid=77851) return next(self.gen) +(EngineCore_DP0 pid=77851) File "/opt/venv/lib64/python3.13/site-packages/vllm/distributed/device_communicators/shm_broadcast.py", line 531, in acquire_read +(EngineCore_DP0 pid=77851) raise RuntimeError("cancelled") +(EngineCore_DP0 pid=77851) RuntimeError: cancelled +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] AsyncLLM output_handler failed. +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [async_llm.py:538] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Error in completion stream generator. +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] Traceback (most recent call last): +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/serving_completion.py", line 352, in completion_stream_generator +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for prompt_idx, res in result_generator: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ...<125 lines>... +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield f"data: {response_json}\n\n" +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/utils/async_utils.py", line 278, in merge_async_iterators +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] async for item in iterators[0]: +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] yield 0, item +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 436, in generate +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] out = q.get_nowait() or await q.get() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/output_processor.py", line 70, in get +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise output +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 490, in output_handler +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] outputs = await engine_core.get_output_async() +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 895, in get_output_async +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] raise self._format_exception(outputs) from None +(APIServer pid=77686) ERROR 12-19 17:38:20 [serving_completion.py:513] vllm.v1.engine.exceptions.EngineDeadError: EngineCore encountered an issue. See stack trace (above) for the root cause. +(APIServer pid=77686) INFO: Shutting down +(APIServer pid=77686) INFO: Waiting for application shutdown. +(APIServer pid=77686) INFO: Application shutdown complete. +(APIServer pid=77686) INFO: Finished server process [77686] +/usr/lib64/python3.13/multiprocessing/resource_tracker.py:324: UserWarning: resource_tracker: There appear to be 2 leaked semaphore objects to clean up at shutdown: {'/mp-zs3r7w6q', '/mp-sdlfr_2q'} + warnings.warn( +/usr/lib64/python3.13/multiprocessing/resource_tracker.py:324: UserWarning: resource_tracker: There appear to be 3 leaked shared_memory objects to clean up at shutdown: {'/psm_0f0e988c', '/psm_760784ea', '/psm_817cd347'} + warnings.warn( diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_throughput.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_throughput.json new file mode 100644 index 0000000..0be91c1 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_throughput.json @@ -0,0 +1,7 @@ +{ + "elapsed_time": 1215.8245900149996, + "num_requests": 1000, + "total_num_tokens": 755432, + "requests_per_second": 0.8224870661545537, + "tokens_per_second": 621.3330493592667 +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json new file mode 100644 index 0000000..3a99a6b --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:19:35 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb7743a2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.40 \nTotal input tokens: 38358 \nTotal generated tokens: 40237 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.79 \nPeak output token throughput (tok/s): 315.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 392.20 \n---------------Time to First Token----------------\nMean TTFT (ms): 125.43 \nMedian TTFT (ms): 115.75 \nP99 TTFT (ms): 233.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 51.71 \nMedian TPOT (ms): 52.62 \nP99 TPOT (ms): 60.97 \n---------------Inter-token Latency----------------\nMean ITL (ms): 51.22 \nMedian ITL (ms): 51.73 \nP99 ITL (ms): 115.89 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..e395862 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:23:04 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-783b8bf2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.51 \nTotal input tokens: 146694 \nTotal generated tokens: 152753 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 612.20 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 171.00 \nTotal token throughput (tok/s): 1200.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 14305.88 \nMedian TTFT (ms): 18966.25 \nP99 TTFT (ms): 31290.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.19 \nMedian TPOT (ms): 90.39 \nP99 TPOT (ms): 108.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.87 \nMedian ITL (ms): 84.16 \nP99 ITL (ms): 193.01 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..69bb2a4 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1038 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 15:18:51 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=59832) INFO 12-19 15:18:51 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=59832) INFO 12-19 15:18:51 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=59832) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=59832) INFO 12-19 15:18:55 [model.py:514] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=59832) INFO 12-19 15:18:55 [model.py:1636] Using max model len 24576 +(APIServer pid=59832) INFO 12-19 15:18:55 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=59999) INFO 12-19 15:18:59 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=59999) INFO 12-19 15:18:59 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:34261 backend=nccl +(EngineCore_DP0 pid=59999) INFO 12-19 15:18:59 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=59999) INFO 12-19 15:19:00 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=59999) INFO 12-19 15:19:00 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=59999) INFO 12-19 15:19:00 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(EngineCore_DP0 pid=59999) INFO 12-19 15:19:00 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=59999) INFO 12-19 15:19:00 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod +(EngineCore_DP0 pid=59999) WARNING 12-19 15:19:00 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=59999) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2904e4f8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.44 \nTotal input tokens: 38358 \nTotal generated tokens: 39896 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 207.32 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal token throughput (tok/s): 406.64 \n---------------Time to First Token----------------\nMean TTFT (ms): 97.06 \nMedian TTFT (ms): 80.47 \nP99 TTFT (ms): 208.68 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 32.90 \nMedian TPOT (ms): 32.05 \nP99 TPOT (ms): 41.21 \n---------------Inter-token Latency----------------\nMean ITL (ms): 32.45 \nMedian ITL (ms): 30.18 \nP99 ITL (ms): 85.86 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json new file mode 100644 index 0000000..3af696c --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 16:53:26 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c9db55f5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 206.61 \nTotal input tokens: 146694 \nTotal generated tokens: 153351 \nRequest throughput (req/s): 3.48 \nOutput token throughput (tok/s): 742.22 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 76.00 \nTotal token throughput (tok/s): 1452.22 \n---------------Time to First Token----------------\nMean TTFT (ms): 194.04 \nMedian TTFT (ms): 127.11 \nP99 TTFT (ms): 1506.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 58.93 \nMedian TPOT (ms): 59.18 \nP99 TPOT (ms): 82.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 58.36 \nMedian ITL (ms): 52.92 \nP99 ITL (ms): 178.94 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log new file mode 100644 index 0000000..755a6a8 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log @@ -0,0 +1,846 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 16:52:23 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=69653) INFO 12-19 16:52:23 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=69653) INFO 12-19 16:52:23 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=69653) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=69653) INFO 12-19 16:52:28 [model.py:514] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=69653) INFO 12-19 16:52:28 [model.py:1636] Using max model len 24576 +(APIServer pid=69653) INFO 12-19 16:52:28 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=69815) INFO 12-19 16:52:32 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=69815) WARNING 12-19 16:52:32 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl +INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl +INFO 12-19 16:52:35 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 16:52:36 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +INFO 12-19 16:52:36 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(Worker_TP0 pid=69898) INFO 12-19 16:52:36 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod +(Worker_TP1 pid=69899) WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod +(Worker_TP0 pid=69898) WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=69898) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00= mamba page size. +(APIServer pid=76251) INFO 12-19 17:11:49 [config.py:463] Padding mamba page size by 1.49% to ensure that mamba page size and attention page size are exactly equal. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=76413) INFO 12-19 17:11:53 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=76413) WARNING 12-19 17:11:53 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl +INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl +INFO 12-19 17:11:57 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 17:11:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +INFO 12-19 17:11:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit... +(Worker_TP1 pid=76496) WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod +(Worker_TP0 pid=76495) WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod +(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=76495) Loading safetensors checkpoint shards: 0% Completed | 0/10 [00:00... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".99", line 333, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".107", line 5, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens], +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ]( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17: +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES: +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks): +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2 +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2") +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2 +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model() +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".99", line 333, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".107", line 5, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens], +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ]( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>... +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns, +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17: +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES: +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load( +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks): +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE) +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2 +(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception. +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2") +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model() +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".99", line 333, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".107", line 5, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens], +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ]( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2") +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model() +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".99", line 333, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File ".107", line 5, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens], +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ]( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>... +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns, +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES: +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load( +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks): +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE) +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2 +(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] EngineCore failed to start. +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] Traceback (most recent call last): +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] super().__init__( +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] vllm_config +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] return aggregate(get_response()) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] raise RuntimeError( +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ...<2 lines>... +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] RuntimeError: Worker failed with error 'at 106:17: +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] if USE_ALIBI_SLOPES: +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] alibi_slope = tl.load( +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] # iterate through tiles +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] for j in range(0, num_blocks): +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] offs_n = tl.arange(0, BLOCK_SIZE) +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^ +(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] arange's range must be a power of 2', please check the stack trace above for the root cause +(EngineCore_DP0 pid=76413) Process EngineCore_DP0: +(EngineCore_DP0 pid=76413) Traceback (most recent call last): +(EngineCore_DP0 pid=76413) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap +(EngineCore_DP0 pid=76413) self.run() +(EngineCore_DP0 pid=76413) ~~~~~~~~^^ +(EngineCore_DP0 pid=76413) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run +(EngineCore_DP0 pid=76413) self._target(*self._args, **self._kwargs) +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +(EngineCore_DP0 pid=76413) raise e +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=76413) engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=76413) super().__init__( +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=76413) vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=76413) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ) +(EngineCore_DP0 pid=76413) ^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=76413) num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=76413) vllm_config +(EngineCore_DP0 pid=76413) ^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) ) +(EngineCore_DP0 pid=76413) ^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=76413) self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=76413) self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=76413) return aggregate(get_response()) +(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=76413) raise RuntimeError( +(EngineCore_DP0 pid=76413) ...<2 lines>... +(EngineCore_DP0 pid=76413) ) +(EngineCore_DP0 pid=76413) RuntimeError: Worker failed with error 'at 106:17: +(EngineCore_DP0 pid=76413) if USE_ALIBI_SLOPES: +(EngineCore_DP0 pid=76413) alibi_slope = tl.load( +(EngineCore_DP0 pid=76413) alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0 +(EngineCore_DP0 pid=76413) ) +(EngineCore_DP0 pid=76413) +(EngineCore_DP0 pid=76413) num_blocks = cdiv_fn(seq_len, BLOCK_SIZE) +(EngineCore_DP0 pid=76413) +(EngineCore_DP0 pid=76413) # iterate through tiles +(EngineCore_DP0 pid=76413) for j in range(0, num_blocks): +(EngineCore_DP0 pid=76413) physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j) +(EngineCore_DP0 pid=76413) +(EngineCore_DP0 pid=76413) offs_n = tl.arange(0, BLOCK_SIZE) +(EngineCore_DP0 pid=76413) ^ +(EngineCore_DP0 pid=76413) arange's range must be a power of 2', please check the stack trace above for the root cause +(Worker_TP0 pid=76495) INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker +(Worker_TP1 pid=76496) INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker +(APIServer pid=76251) Traceback (most recent call last): +(APIServer pid=76251) File "/opt/venv/bin/vllm", line 7, in +(APIServer pid=76251) sys.exit(main()) +(APIServer pid=76251) ~~~~^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main +(APIServer pid=76251) args.dispatch_function(args) +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd +(APIServer pid=76251) uvloop.run(run_server(args)) +(APIServer pid=76251) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run +(APIServer pid=76251) return __asyncio.run( +(APIServer pid=76251) ~~~~~~~~~~~~~^ +(APIServer pid=76251) wrapper(), +(APIServer pid=76251) ^^^^^^^^^^ +(APIServer pid=76251) ...<2 lines>... +(APIServer pid=76251) **run_kwargs +(APIServer pid=76251) ^^^^^^^^^^^^ +(APIServer pid=76251) ) +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run +(APIServer pid=76251) return runner.run(main) +(APIServer pid=76251) ~~~~~~~~~~^^^^^^ +(APIServer pid=76251) File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run +(APIServer pid=76251) return self._loop.run_until_complete(task) +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=76251) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper +(APIServer pid=76251) return await main +(APIServer pid=76251) ^^^^^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server +(APIServer pid=76251) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker +(APIServer pid=76251) async with build_async_engine_client( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=76251) args, +(APIServer pid=76251) ^^^^^ +(APIServer pid=76251) client_config=client_config, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ) as engine_client: +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=76251) return await anext(self.gen) +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client +(APIServer pid=76251) async with build_async_engine_client_from_engine_args( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=76251) engine_args, +(APIServer pid=76251) ^^^^^^^^^^^^ +(APIServer pid=76251) ...<2 lines>... +(APIServer pid=76251) client_config=client_config, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ) as engine: +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=76251) return await anext(self.gen) +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args +(APIServer pid=76251) async_llm = AsyncLLM.from_vllm_config( +(APIServer pid=76251) vllm_config=vllm_config, +(APIServer pid=76251) ...<6 lines>... +(APIServer pid=76251) client_index=client_index, +(APIServer pid=76251) ) +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config +(APIServer pid=76251) return cls( +(APIServer pid=76251) vllm_config=vllm_config, +(APIServer pid=76251) ...<9 lines>... +(APIServer pid=76251) client_index=client_index, +(APIServer pid=76251) ) +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__ +(APIServer pid=76251) self.engine_core = EngineCoreClient.make_async_mp_client( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=76251) vllm_config=vllm_config, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ...<4 lines>... +(APIServer pid=76251) client_index=client_index, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ) +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client +(APIServer pid=76251) return AsyncMPClient(*client_args) +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__ +(APIServer pid=76251) super().__init__( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~^ +(APIServer pid=76251) asyncio_mode=True, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ...<3 lines>... +(APIServer pid=76251) client_addresses=client_addresses, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ) +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 477, in __init__ +(APIServer pid=76251) with launch_core_engines(vllm_config, executor_class, log_stats) as ( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) File "/usr/lib64/python3.13/contextlib.py", line 148, in __exit__ +(APIServer pid=76251) next(self.gen) +(APIServer pid=76251) ~~~~^^^^^^^^^^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 903, in launch_core_engines +(APIServer pid=76251) wait_for_engine_startup( +(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=76251) handshake_socket, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ...<5 lines>... +(APIServer pid=76251) coordinator.proc if coordinator else None, +(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=76251) ) +(APIServer pid=76251) ^ +(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup +(APIServer pid=76251) raise RuntimeError( +(APIServer pid=76251) ...<3 lines>... +(APIServer pid=76251) ) +(APIServer pid=76251) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json new file mode 100644 index 0000000..3eabad4 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 14:23:22 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af26c427-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38794 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.66 \nPeak output token throughput (tok/s): 398.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 392.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.71 \nMedian TTFT (ms): 61.84 \nP99 TTFT (ms): 180.02 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.85 \nMedian TPOT (ms): 30.79 \nP99 TPOT (ms): 33.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.86 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 55.30 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..2813518 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 14:26:47 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-65595a03-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.73 \nTotal input tokens: 145810 \nTotal generated tokens: 152105 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 742.97 \nPeak output token throughput (tok/s): 1252.00 \nPeak concurrent requests: 55.00 \nTotal token throughput (tok/s): 1455.19 \n---------------Time to First Token----------------\nMean TTFT (ms): 81.58 \nMedian TTFT (ms): 70.11 \nP99 TTFT (ms): 184.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.28 \nMedian TPOT (ms): 36.83 \nP99 TPOT (ms): 50.83 \n---------------Inter-token Latency----------------\nMean ITL (ms): 36.98 \nMedian ITL (ms): 34.37 \nP99 ITL (ms): 119.18 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..3492884 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1027 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 14:22:47 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=56561) INFO 12-19 14:22:47 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=56561) INFO 12-19 14:22:47 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=56561) INFO 12-19 14:22:51 [model.py:514] Resolved architecture: LlamaForCausalLM +(APIServer pid=56561) INFO 12-19 14:22:51 [model.py:1636] Using max model len 65536 +(APIServer pid=56561) INFO 12-19 14:22:51 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=56723) INFO 12-19 14:22:56 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=56723) INFO 12-19 14:22:56 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:59373 backend=nccl +(EngineCore_DP0 pid=56723) INFO 12-19 14:22:56 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=56723) INFO 12-19 14:22:56 [gpu_model_runner.py:3562] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=56723) INFO 12-19 14:22:56 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(EngineCore_DP0 pid=56723) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-819f7ea7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 189.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 204.80 \nPeak output token throughput (tok/s): 415.00 \nPeak concurrent requests: 10.00 \nTotal token throughput (tok/s): 404.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.83 \nMedian TTFT (ms): 50.95 \nP99 TTFT (ms): 232.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.52 \nMedian TPOT (ms): 19.18 \nP99 TPOT (ms): 24.48 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.42 \nMedian ITL (ms): 18.84 \nP99 ITL (ms): 34.36 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json new file mode 100644 index 0000000..ef40343 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 15:44:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-9c7b74a1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.04 \nTotal input tokens: 145810 \nTotal generated tokens: 152194 \nRequest throughput (req/s): 3.67 \nOutput token throughput (tok/s): 776.36 \nPeak output token throughput (tok/s): 1303.00 \nPeak concurrent requests: 46.00 \nTotal token throughput (tok/s): 1520.15 \n---------------Time to First Token----------------\nMean TTFT (ms): 86.17 \nMedian TTFT (ms): 57.78 \nP99 TTFT (ms): 291.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.21 \nMedian TPOT (ms): 27.26 \nP99 TPOT (ms): 51.74 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.56 \nMedian ITL (ms): 23.38 \nP99 ITL (ms): 140.32 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log new file mode 100644 index 0000000..1849592 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log @@ -0,0 +1,1034 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 15:40:52 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=62066) INFO 12-19 15:40:52 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=62066) INFO 12-19 15:40:52 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=62066) INFO 12-19 15:40:56 [model.py:514] Resolved architecture: LlamaForCausalLM +(APIServer pid=62066) INFO 12-19 15:40:56 [model.py:1636] Using max model len 65536 +(APIServer pid=62066) INFO 12-19 15:40:57 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +(EngineCore_DP0 pid=62228) INFO 12-19 15:41:01 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=62228) WARNING 12-19 15:41:01 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 15:41:04 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:40771 backend=nccl +INFO 12-19 15:41:04 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:40771 backend=nccl +INFO 12-19 15:41:04 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 15:41:05 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-19 15:41:05 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=62310) INFO 12-19 15:41:05 [gpu_model_runner.py:3562] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(Worker_TP1 pid=62311) INFO 12-19 15:41:06 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=62310) INFO 12-19 15:41:06 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=62310) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3e31d4e4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 215.02 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.28 \nPeak output token throughput (tok/s): 324.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 362.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 118.15 \nMedian TTFT (ms): 106.93 \nP99 TTFT (ms): 227.27 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.56 \nMedian TPOT (ms): 55.08 \nP99 TPOT (ms): 69.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.82 \nMedian ITL (ms): 54.23 \nP99 ITL (ms): 120.82 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..6192364 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "WARNING 12-19 14:45:51 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84255adf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 238.83 \nTotal input tokens: 145540 \nTotal generated tokens: 151955 \nRequest throughput (req/s): 3.01 \nOutput token throughput (tok/s): 636.24 \nPeak output token throughput (tok/s): 960.00 \nPeak concurrent requests: 94.00 \nTotal token throughput (tok/s): 1245.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 1170.97 \nMedian TTFT (ms): 182.43 \nP99 TTFT (ms): 5964.07 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.13 \nMedian TPOT (ms): 74.02 \nP99 TPOT (ms): 87.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 72.97 \nMedian ITL (ms): 68.87 \nP99 ITL (ms): 158.23 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..da9466a --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1033 @@ +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +WARNING 12-19 14:41:24 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead. +(APIServer pid=57644) INFO 12-19 14:41:24 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213 +(APIServer pid=57644) INFO 12-19 14:41:24 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=57644) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=57644) INFO 12-19 14:41:28 [model.py:514] Resolved architecture: GptOssForCausalLM +(APIServer pid=57644) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:33 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:59097 backend=nccl +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:33 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:34 [gpu_model_runner.py:3562] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:34 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:34 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=57811) INFO 12-19 14:41:34 [mxfp4.py:170] Using Triton backend +(EngineCore_DP0 pid=57811) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=75118) WARNING 12-19 17:08:00 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info +INFO 12-19 17:08:03 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:58633 backend=nccl +INFO 12-19 17:08:03 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:58633 backend=nccl +INFO 12-19 17:08:04 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-19 17:08:04 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +INFO 12-19 17:08:04 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(Worker_TP0 pid=75200) INFO 12-19 17:08:05 [gpu_model_runner.py:3562] Starting to load model openai/gpt-oss-20b... +(Worker_TP1 pid=75201) INFO 12-19 17:08:05 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP1 pid=75201) INFO 12-19 17:08:05 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=75201) INFO 12-19 17:08:05 [mxfp4.py:170] Using Triton backend +(Worker_TP0 pid=75200) INFO 12-19 17:08:05 [rocm.py:306] Using Rocm Attention backend on V1 engine. +(Worker_TP0 pid=75200) INFO 12-19 17:08:05 [layer.py:372] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=75200) INFO 12-19 17:08:05 [mxfp4.py:170] Using Triton backend +(Worker_TP0 pid=75200) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00... +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 150.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.76 GiB is allocated by PyTorch, with 106.00 MiB allocated in private pools (e.g., HIP Graphs), and 135.04 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) from e +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] WorkerProc hit an exception. +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits, +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 150.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.76 GiB is allocated by PyTorch, with 106.00 MiB allocated in private pools (e.g., HIP Graphs), and 119.04 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) from e +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits, +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 150.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.76 GiB is allocated by PyTorch, with 106.00 MiB allocated in private pools (e.g., HIP Graphs), and 135.04 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) from e +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP0 pid=75200) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampler_output = self.sampler( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits, +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<2 lines>... +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] sampling_metadata.top_p, +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return forward_call(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 150.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.76 GiB is allocated by PyTorch, with 106.00 MiB allocated in private pools (e.g., HIP Graphs), and 119.04 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] The above exception was the direct cause of the following exception: +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] Traceback (most recent call last): +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] output = func(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] return func(*args, **kwargs) +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] raise RuntimeError( +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ...<4 lines>... +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] ) from e +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine. +(Worker_TP1 pid=75201) ERROR 12-19 17:08:43 [multiproc_executor.py:826] +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] EngineCore failed to start. +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] Traceback (most recent call last): +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] super().__init__( +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] vllm_config +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] return aggregate(get_response()) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] raise RuntimeError( +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ...<2 lines>... +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] ) +(EngineCore_DP0 pid=75118) ERROR 12-19 17:08:43 [core.py:866] RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause +(EngineCore_DP0 pid=75118) Process EngineCore_DP0: +(EngineCore_DP0 pid=75118) Traceback (most recent call last): +(EngineCore_DP0 pid=75118) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap +(EngineCore_DP0 pid=75118) self.run() +(EngineCore_DP0 pid=75118) ~~~~~~~~^^ +(EngineCore_DP0 pid=75118) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run +(EngineCore_DP0 pid=75118) self._target(*self._args, **self._kwargs) +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core +(EngineCore_DP0 pid=75118) raise e +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core +(EngineCore_DP0 pid=75118) engine_core = EngineCoreProc(*args, **kwargs) +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__ +(EngineCore_DP0 pid=75118) super().__init__( +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75118) vllm_config, executor_class, log_stats, executor_fail_callback +(EngineCore_DP0 pid=75118) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ) +(EngineCore_DP0 pid=75118) ^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__ +(EngineCore_DP0 pid=75118) num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(EngineCore_DP0 pid=75118) vllm_config +(EngineCore_DP0 pid=75118) ^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) ) +(EngineCore_DP0 pid=75118) ^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches +(EngineCore_DP0 pid=75118) self.model_executor.initialize_from_config(kv_cache_configs) +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config +(EngineCore_DP0 pid=75118) self.collective_rpc("compile_or_warm_up_model") +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc +(EngineCore_DP0 pid=75118) return aggregate(get_response()) +(EngineCore_DP0 pid=75118) ~~~~~~~~~~~~^^ +(EngineCore_DP0 pid=75118) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response +(EngineCore_DP0 pid=75118) raise RuntimeError( +(EngineCore_DP0 pid=75118) ...<2 lines>... +(EngineCore_DP0 pid=75118) ) +(EngineCore_DP0 pid=75118) RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause +(Worker_TP0 pid=75200) INFO 12-19 17:08:43 [multiproc_executor.py:711] Parent process exited, terminating worker +(Worker_TP1 pid=75201) INFO 12-19 17:08:43 [multiproc_executor.py:711] Parent process exited, terminating worker +(APIServer pid=74952) Traceback (most recent call last): +(APIServer pid=74952) File "/opt/venv/bin/vllm", line 7, in +(APIServer pid=74952) sys.exit(main()) +(APIServer pid=74952) ~~~~^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main +(APIServer pid=74952) args.dispatch_function(args) +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd +(APIServer pid=74952) uvloop.run(run_server(args)) +(APIServer pid=74952) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run +(APIServer pid=74952) return __asyncio.run( +(APIServer pid=74952) ~~~~~~~~~~~~~^ +(APIServer pid=74952) wrapper(), +(APIServer pid=74952) ^^^^^^^^^^ +(APIServer pid=74952) ...<2 lines>... +(APIServer pid=74952) **run_kwargs +(APIServer pid=74952) ^^^^^^^^^^^^ +(APIServer pid=74952) ) +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run +(APIServer pid=74952) return runner.run(main) +(APIServer pid=74952) ~~~~~~~~~~^^^^^^ +(APIServer pid=74952) File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run +(APIServer pid=74952) return self._loop.run_until_complete(task) +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ +(APIServer pid=74952) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper +(APIServer pid=74952) return await main +(APIServer pid=74952) ^^^^^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server +(APIServer pid=74952) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker +(APIServer pid=74952) async with build_async_engine_client( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=74952) args, +(APIServer pid=74952) ^^^^^ +(APIServer pid=74952) client_config=client_config, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ) as engine_client: +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=74952) return await anext(self.gen) +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client +(APIServer pid=74952) async with build_async_engine_client_from_engine_args( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=74952) engine_args, +(APIServer pid=74952) ^^^^^^^^^^^^ +(APIServer pid=74952) ...<2 lines>... +(APIServer pid=74952) client_config=client_config, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ) as engine: +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__ +(APIServer pid=74952) return await anext(self.gen) +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args +(APIServer pid=74952) async_llm = AsyncLLM.from_vllm_config( +(APIServer pid=74952) vllm_config=vllm_config, +(APIServer pid=74952) ...<6 lines>... +(APIServer pid=74952) client_index=client_index, +(APIServer pid=74952) ) +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config +(APIServer pid=74952) return cls( +(APIServer pid=74952) vllm_config=vllm_config, +(APIServer pid=74952) ...<9 lines>... +(APIServer pid=74952) client_index=client_index, +(APIServer pid=74952) ) +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__ +(APIServer pid=74952) self.engine_core = EngineCoreClient.make_async_mp_client( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=74952) vllm_config=vllm_config, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ...<4 lines>... +(APIServer pid=74952) client_index=client_index, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ) +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client +(APIServer pid=74952) return AsyncMPClient(*client_args) +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__ +(APIServer pid=74952) super().__init__( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~^ +(APIServer pid=74952) asyncio_mode=True, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ...<3 lines>... +(APIServer pid=74952) client_addresses=client_addresses, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ) +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 477, in __init__ +(APIServer pid=74952) with launch_core_engines(vllm_config, executor_class, log_stats) as ( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) File "/usr/lib64/python3.13/contextlib.py", line 148, in __exit__ +(APIServer pid=74952) next(self.gen) +(APIServer pid=74952) ~~~~^^^^^^^^^^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 903, in launch_core_engines +(APIServer pid=74952) wait_for_engine_startup( +(APIServer pid=74952) ~~~~~~~~~~~~~~~~~~~~~~~^ +(APIServer pid=74952) handshake_socket, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ...<5 lines>... +(APIServer pid=74952) coordinator.proc if coordinator else None, +(APIServer pid=74952) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=74952) ) +(APIServer pid=74952) ^ +(APIServer pid=74952) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/utils.py", line 960, in wait_for_engine_startup +(APIServer pid=74952) raise RuntimeError( +(APIServer pid=74952) ...<3 lines>... +(APIServer pid=74952) ) +(APIServer pid=74952) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json new file mode 100644 index 0000000..acdb5fe --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e4e803cf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.63 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.85 \nPeak output token throughput (tok/s): 350.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 392.04 \n---------------Time to First Token----------------\nMean TTFT (ms): 100.59 \nMedian TTFT (ms): 84.87 \nP99 TTFT (ms): 217.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.46 \nMedian TPOT (ms): 41.30 \nP99 TPOT (ms): 45.56 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.46 \nMedian ITL (ms): 40.41 \nP99 ITL (ms): 81.46 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..fe1d0b5 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2d3278b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 211.85 \nTotal input tokens: 146694 \nTotal generated tokens: 155558 \nRequest throughput (req/s): 3.40 \nOutput token throughput (tok/s): 734.30 \nPeak output token throughput (tok/s): 1241.00 \nPeak concurrent requests: 68.00 \nTotal Token throughput (tok/s): 1426.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 111.24 \nMedian TTFT (ms): 89.87 \nP99 TTFT (ms): 307.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 52.80 \nMedian TPOT (ms): 52.89 \nP99 TPOT (ms): 76.85 \n---------------Inter-token Latency----------------\nMean ITL (ms): 52.36 \nMedian ITL (ms): 48.02 \nP99 ITL (ms): 172.90 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..a5a823a --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1042 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=15688) INFO 12-12 17:38:31 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=15688) INFO 12-12 17:38:31 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=15688) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=15688) INFO 12-12 17:38:35 [model.py:629] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=15688) INFO 12-12 17:38:35 [model.py:1755] Using max model len 32768 +(APIServer pid=15688) INFO 12-12 17:38:35 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=15850) INFO 12-12 17:38:39 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=15850) INFO 12-12 17:38:39 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:56369 backend=nccl +(EngineCore_DP0 pid=15850) INFO 12-12 17:38:39 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=15850) INFO 12-12 17:38:39 [gpu_model_runner.py:3544] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=15850) INFO 12-12 17:38:40 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=15850) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fce2ee75-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.29 \nTotal input tokens: 37841 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.79 \nPeak output token throughput (tok/s): 406.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.56 \n---------------Time to First Token----------------\nMean TTFT (ms): 73.39 \nMedian TTFT (ms): 61.33 \nP99 TTFT (ms): 185.43 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.73 \nMedian TPOT (ms): 30.60 \nP99 TPOT (ms): 33.54 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.74 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 50.56 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..d9912a2 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af5860ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152213 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 744.74 \nPeak output token throughput (tok/s): 1238.00 \nPeak concurrent requests: 53.00 \nTotal Token throughput (tok/s): 1458.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.15 \nMedian TTFT (ms): 63.63 \nP99 TTFT (ms): 179.56 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 35.83 \nMedian TPOT (ms): 35.55 \nP99 TPOT (ms): 48.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.60 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 112.07 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..b7dfd8d --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1040 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=8444) INFO 12-12 16:58:44 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=8444) INFO 12-12 16:58:44 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=8444) INFO 12-12 16:58:48 [model.py:629] Resolved architecture: LlamaForCausalLM +(APIServer pid=8444) INFO 12-12 16:58:48 [model.py:1755] Using max model len 65536 +(APIServer pid=8444) INFO 12-12 16:58:48 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=8607) INFO 12-12 16:58:52 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=8607) INFO 12-12 16:58:52 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:38471 backend=nccl +(EngineCore_DP0 pid=8607) INFO 12-12 16:58:52 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=8607) INFO 12-12 16:58:52 [gpu_model_runner.py:3544] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=8607) INFO 12-12 16:58:52 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=8607) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1dd28fd7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 214.85 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.43 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 19.00 \nTotal Token throughput (tok/s): 362.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 115.51 \nMedian TTFT (ms): 108.55 \nP99 TTFT (ms): 235.09 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 53.14 \nMedian TPOT (ms): 53.86 \nP99 TPOT (ms): 69.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 53.53 \nMedian ITL (ms): 52.81 \nP99 ITL (ms): 120.56 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..788910c --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f4f4694-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 236.39 \nTotal input tokens: 145540 \nTotal generated tokens: 152178 \nRequest throughput (req/s): 3.05 \nOutput token throughput (tok/s): 643.77 \nPeak output token throughput (tok/s): 1024.00 \nPeak concurrent requests: 92.00 \nTotal Token throughput (tok/s): 1259.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 850.20 \nMedian TTFT (ms): 141.29 \nP99 TTFT (ms): 5024.92 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.13 \nMedian TPOT (ms): 71.70 \nP99 TPOT (ms): 89.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.90 \nMedian ITL (ms): 66.80 \nP99 ITL (ms): 157.64 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..6ea9f83 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1045 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=13351) INFO 12-12 17:18:27 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=13351) INFO 12-12 17:18:27 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=13351) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=13351) INFO 12-12 17:18:31 [model.py:629] Resolved architecture: GptOssForCausalLM +(APIServer pid=13351) Parse safetensors files: 0%| | 0/3 [00:00 Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='openai/gpt-oss-20b', speculative_config=None, tokenizer='openai/gpt-oss-20b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=mxfp4, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='openai_gptoss', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=openai/gpt-oss-20b, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:47683 backend=nccl +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [gpu_model_runner.py:3544] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=13518) INFO 12-12 17:18:36 [mxfp4.py:171] Using Triton backend +(EngineCore_DP0 pid=13518) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84569f25-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 203.97 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.88 \nOutput token throughput (tok/s): 197.56 \nPeak output token throughput (tok/s): 330.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 385.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 108.91 \nMedian TTFT (ms): 95.15 \nP99 TTFT (ms): 230.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.90 \nMedian TPOT (ms): 47.66 \nP99 TPOT (ms): 53.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.94 \nMedian ITL (ms): 46.64 \nP99 ITL (ms): 85.76 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..12f0a86 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a714d425-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 217.26 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.31 \nOutput token throughput (tok/s): 716.13 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 77.00 \nTotal Token throughput (tok/s): 1391.33 \n---------------Time to First Token----------------\nMean TTFT (ms): 186.47 \nMedian TTFT (ms): 105.59 \nP99 TTFT (ms): 1603.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 59.68 \nMedian TPOT (ms): 59.91 \nP99 TPOT (ms): 84.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 59.35 \nMedian ITL (ms): 54.32 \nP99 ITL (ms): 190.63 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..7dc483e --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1047 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=11550) INFO 12-09 19:31:49 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=11550) INFO 12-09 19:31:49 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=11550) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=11550) INFO 12-09 19:31:53 [model.py:629] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=11550) INFO 12-09 19:31:53 [model.py:1755] Using max model len 32768 +(APIServer pid=11550) INFO 12-09 19:31:53 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=11713) INFO 12-09 19:31:57 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=11713) INFO 12-09 19:31:57 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:56787 backend=nccl +(EngineCore_DP0 pid=11713) INFO 12-09 19:31:57 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=11713) INFO 12-09 19:31:58 [gpu_model_runner.py:3544] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=11713) INFO 12-09 19:31:58 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=11713) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a3fea59e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.99 \nTotal input tokens: 40429 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 197.07 \nPeak output token throughput (tok/s): 340.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 402.31 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.59 \nMedian TTFT (ms): 82.18 \nP99 TTFT (ms): 271.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 38.86 \nMedian TPOT (ms): 38.92 \nP99 TPOT (ms): 50.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.20 \nMedian ITL (ms): 32.74 \nP99 ITL (ms): 86.49 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..ee001de --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f36abb16-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 241.29 \nTotal input tokens: 158086 \nTotal generated tokens: 152183 \nRequest throughput (req/s): 2.98 \nOutput token throughput (tok/s): 630.70 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 152.00 \nTotal Token throughput (tok/s): 1285.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 12436.56 \nMedian TTFT (ms): 17115.90 \nP99 TTFT (ms): 24750.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 84.04 \nMedian TPOT (ms): 85.32 \nP99 TPOT (ms): 115.82 \n---------------Inter-token Latency----------------\nMean ITL (ms): 83.72 \nMedian ITL (ms): 75.54 \nP99 ITL (ms): 261.14 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..c03a57f --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log @@ -0,0 +1,1048 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=5124) INFO 12-11 18:45:29 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=5124) INFO 12-11 18:45:29 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=5124) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=5124) INFO 12-11 18:45:33 [model.py:629] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=5124) INFO 12-11 18:45:33 [model.py:1755] Using max model len 32768 +(APIServer pid=5124) INFO 12-11 18:45:33 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:38 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='RedHatAI/gemma-3-12b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-12b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-12b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:40 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:45401 backend=nccl +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:40 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=5288) Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:45 [gpu_model_runner.py:3544] Starting to load model RedHatAI/gemma-3-12b-it-FP8-dynamic... +(EngineCore_DP0 pid=5288) WARNING 12-11 18:45:46 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:46 [layer.py:524] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(EngineCore_DP0 pid=5288) WARNING 12-11 18:45:46 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(EngineCore_DP0 pid=5288) INFO 12-11 18:45:46 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=5288) WARNING 12-11 18:45:46 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(EngineCore_DP0 pid=5288) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a1348ed0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.66 \nTotal input tokens: 40429 \nTotal generated tokens: 39127 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 205.22 \nPeak output token throughput (tok/s): 422.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 417.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 89.11 \nMedian TTFT (ms): 64.29 \nP99 TTFT (ms): 255.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.00 \nMedian TPOT (ms): 21.60 \nP99 TPOT (ms): 28.81 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.91 \nMedian ITL (ms): 20.88 \nP99 ITL (ms): 45.88 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json new file mode 100644 index 0000000..fd03801 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-370f24dd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.05 \nTotal input tokens: 158086 \nTotal generated tokens: 153200 \nRequest throughput (req/s): 3.55 \nOutput token throughput (tok/s): 754.51 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 78.00 \nTotal Token throughput (tok/s): 1533.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 222.29 \nMedian TTFT (ms): 103.65 \nP99 TTFT (ms): 2503.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.74 \nMedian TPOT (ms): 55.60 \nP99 TPOT (ms): 83.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.25 \nMedian ITL (ms): 49.31 \nP99 ITL (ms): 204.91 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log new file mode 100644 index 0000000..1afb9aa --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log @@ -0,0 +1,1074 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=10267) INFO 12-11 19:27:42 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=10267) INFO 12-11 19:27:42 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-12b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 9900, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=10267) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=10267) INFO 12-11 19:27:46 [model.py:629] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=10267) INFO 12-11 19:27:46 [model.py:1755] Using max model len 9900 +(APIServer pid=10267) INFO 12-11 19:27:46 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=10431) INFO 12-11 19:27:51 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='RedHatAI/gemma-3-12b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-12b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=9900, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-12b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=10431) WARNING 12-11 19:27:51 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-11 19:27:56 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:56779 backend=nccl +INFO 12-11 19:27:56 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:56779 backend=nccl +INFO 12-11 19:27:56 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-11 19:27:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-11 19:27:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(Worker_TP0 pid=10513) INFO 12-11 19:28:03 [gpu_model_runner.py:3544] Starting to load model RedHatAI/gemma-3-12b-it-FP8-dynamic... +(Worker_TP1 pid=10514) WARNING 12-11 19:28:03 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=10514) INFO 12-11 19:28:03 [layer.py:524] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP1 pid=10514) WARNING 12-11 19:28:03 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP1 pid=10514) INFO 12-11 19:28:03 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP1 pid=10514) WARNING 12-11 19:28:03 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=10513) WARNING 12-11 19:28:03 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=10513) INFO 12-11 19:28:03 [layer.py:524] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP0 pid=10513) WARNING 12-11 19:28:03 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP0 pid=10513) INFO 12-11 19:28:03 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=10513) WARNING 12-11 19:28:03 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=10513) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb2c00d0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.25 \nTotal input tokens: 40429 \nTotal generated tokens: 39398 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 194.80 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 17.00 \nTotal Token throughput (tok/s): 394.70 \n---------------Time to First Token----------------\nMean TTFT (ms): 135.44 \nMedian TTFT (ms): 102.54 \nP99 TTFT (ms): 390.74 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.24 \nMedian TPOT (ms): 46.60 \nP99 TPOT (ms): 56.08 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.00 \nMedian ITL (ms): 45.58 \nP99 ITL (ms): 140.61 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json new file mode 100644 index 0000000..1c37ec0 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb0dc17a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 320.24 \nTotal input tokens: 158086 \nTotal generated tokens: 152563 \nRequest throughput (req/s): 2.25 \nOutput token throughput (tok/s): 476.40 \nPeak output token throughput (tok/s): 608.00 \nPeak concurrent requests: 304.00 \nTotal Token throughput (tok/s): 970.05 \n---------------Time to First Token----------------\nMean TTFT (ms): 52928.91 \nMedian TTFT (ms): 59992.28 \nP99 TTFT (ms): 104765.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.40 \nMedian TPOT (ms): 60.27 \nP99 TPOT (ms): 90.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 60.58 \nMedian ITL (ms): 53.74 \nP99 ITL (ms): 244.74 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log new file mode 100644 index 0000000..67be5ec --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log @@ -0,0 +1,1092 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=12756) INFO 12-10 16:47:57 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=12756) INFO 12-10 16:47:57 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/gemma-3-27b-it-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/gemma-3-27b-it-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 29000, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.94, 'max_num_seqs': 32} +(APIServer pid=12756) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=12756) INFO 12-10 16:48:02 [model.py:629] Resolved architecture: Gemma3ForConditionalGeneration +(APIServer pid=12756) INFO 12-10 16:48:02 [model.py:1755] Using max model len 29000 +(APIServer pid=12756) INFO 12-10 16:48:02 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=12920) INFO 12-10 16:48:07 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='RedHatAI/gemma-3-27b-it-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/gemma-3-27b-it-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=29000, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/gemma-3-27b-it-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=12920) WARNING 12-10 16:48:07 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-10 16:48:11 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:41237 backend=nccl +INFO 12-10 16:48:11 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:41237 backend=nccl +INFO 12-10 16:48:11 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-10 16:48:12 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-10 16:48:12 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +(Worker_TP0 pid=13002) INFO 12-10 16:48:18 [gpu_model_runner.py:3544] Starting to load model RedHatAI/gemma-3-27b-it-FP8-dynamic... +(Worker_TP1 pid=13003) WARNING 12-10 16:48:18 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=13003) INFO 12-10 16:48:18 [layer.py:524] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP1 pid=13003) WARNING 12-10 16:48:18 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP1 pid=13003) INFO 12-10 16:48:18 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP1 pid=13003) WARNING 12-10 16:48:18 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=13002) WARNING 12-10 16:48:18 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=13002) INFO 12-10 16:48:18 [layer.py:524] Using AttentionBackendEnum.TORCH_SDPA for MultiHeadAttention in multimodal encoder. +(Worker_TP0 pid=13002) WARNING 12-10 16:48:18 [activation.py:544] [ROCm] PyTorch's native GELU with tanh approximation is unstable. Falling back to GELU(approximate='none'). +(Worker_TP0 pid=13002) INFO 12-10 16:48:18 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=13002) WARNING 12-10 16:48:18 [activation.py:220] [ROCm] PyTorch's native GELU with tanh approximation is unstable with torch.compile. For native implementation, fallback to 'none' approximation. The custom kernel implementation is unaffected. +(Worker_TP0 pid=13002) Loading safetensors checkpoint shards: 0% Completed | 0/6 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1e0af39c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40157 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 202.57 \nPeak output token throughput (tok/s): 311.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 396.07 \n---------------Time to First Token----------------\nMean TTFT (ms): 120.83 \nMedian TTFT (ms): 107.92 \nP99 TTFT (ms): 215.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.17 \nMedian TPOT (ms): 44.69 \nP99 TPOT (ms): 56.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.94 \nMedian ITL (ms): 42.97 \nP99 ITL (ms): 110.51 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..eeae5b3 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5a7df568-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.37 \nTotal input tokens: 146694 \nTotal generated tokens: 153158 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 614.17 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 168.00 \nTotal Token throughput (tok/s): 1202.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 14118.91 \nMedian TTFT (ms): 18607.33 \nP99 TTFT (ms): 31435.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.10 \nMedian TPOT (ms): 90.68 \nP99 TPOT (ms): 104.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.93 \nMedian ITL (ms): 84.69 \nP99 ITL (ms): 190.09 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..251da92 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1054 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=16712) INFO 12-09 19:55:48 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=16712) INFO 12-09 19:55:48 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=16712) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=16712) INFO 12-09 19:55:52 [model.py:629] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=16712) INFO 12-09 19:55:52 [model.py:1755] Using max model len 24576 +(APIServer pid=16712) INFO 12-09 19:55:52 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:56 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:56 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:38381 backend=nccl +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:56 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:57 [gpu_model_runner.py:3544] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:57 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:57 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:57 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=16880) INFO 12-09 19:55:57 [compressed_tensors_moe.py:189] Using CompressedTensorsWNA16MoEMethod +(EngineCore_DP0 pid=16880) WARNING 12-09 19:55:57 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=16880) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df1894e5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.60 \nTotal input tokens: 38358 \nTotal generated tokens: 39514 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 206.23 \nPeak output token throughput (tok/s): 361.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 406.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.14 \nMedian TTFT (ms): 66.00 \nP99 TTFT (ms): 145.78 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.00 \nMedian TPOT (ms): 28.14 \nP99 TPOT (ms): 35.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.59 \nMedian ITL (ms): 27.66 \nP99 ITL (ms): 54.69 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json new file mode 100644 index 0000000..78c75b6 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e434292b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.64 \nTotal input tokens: 146694 \nTotal generated tokens: 153347 \nRequest throughput (req/s): 3.54 \nOutput token throughput (tok/s): 753.01 \nPeak output token throughput (tok/s): 1160.00 \nPeak concurrent requests: 65.00 \nTotal Token throughput (tok/s): 1473.36 \n---------------Time to First Token----------------\nMean TTFT (ms): 93.13 \nMedian TTFT (ms): 87.23 \nP99 TTFT (ms): 181.31 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.81 \nMedian TPOT (ms): 47.62 \nP99 TPOT (ms): 61.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.43 \nMedian ITL (ms): 44.70 \nP99 ITL (ms): 109.90 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log new file mode 100644 index 0000000..d5eea1f --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log @@ -0,0 +1,1076 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=43391) INFO 12-09 20:50:38 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=43391) INFO 12-09 20:50:38 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=43391) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=43391) INFO 12-09 20:50:42 [model.py:629] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=43391) INFO 12-09 20:50:42 [model.py:1755] Using max model len 24576 +(APIServer pid=43391) INFO 12-09 20:50:42 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=43554) INFO 12-09 20:50:46 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=43554) WARNING 12-09 20:50:46 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-09 20:50:49 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:43809 backend=nccl +INFO 12-09 20:50:49 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:43809 backend=nccl +INFO 12-09 20:50:49 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-09 20:50:50 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-09 20:50:50 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=43636) INFO 12-09 20:50:50 [gpu_model_runner.py:3544] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(Worker_TP1 pid=43637) INFO 12-09 20:50:50 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(Worker_TP0 pid=43636) INFO 12-09 20:50:50 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16 +(Worker_TP1 pid=43637) INFO 12-09 20:50:50 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP1 pid=43637) INFO 12-09 20:50:50 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=43637) INFO 12-09 20:50:50 [compressed_tensors_moe.py:189] Using CompressedTensorsWNA16MoEMethod +(Worker_TP1 pid=43637) WARNING 12-09 20:50:50 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=43636) INFO 12-09 20:50:50 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=43636) INFO 12-09 20:50:50 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=43636) INFO 12-09 20:50:50 [compressed_tensors_moe.py:189] Using CompressedTensorsWNA16MoEMethod +(Worker_TP0 pid=43636) WARNING 12-09 20:50:50 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=43636) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b53e2ad8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.26 \nTotal input tokens: 38358 \nTotal generated tokens: 38404 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 195.68 \nPeak output token throughput (tok/s): 304.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 391.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 519.96 \nMedian TTFT (ms): 124.31 \nP99 TTFT (ms): 10120.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 44.04 \nMedian TPOT (ms): 42.30 \nP99 TPOT (ms): 57.88 \n---------------Inter-token Latency----------------\nMean ITL (ms): 43.67 \nMedian ITL (ms): 40.54 \nP99 ITL (ms): 125.33 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_qps4.0_latency.json new file mode 100644 index 0000000..1dc37a9 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-72f2dd3c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 351.46 \nTotal input tokens: 146694 \nTotal generated tokens: 148389 \nRequest throughput (req/s): 2.05 \nOutput token throughput (tok/s): 422.20 \nPeak output token throughput (tok/s): 544.00 \nPeak concurrent requests: 369.00 \nTotal Token throughput (tok/s): 839.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 72209.01 \nMedian TTFT (ms): 79268.59 \nP99 TTFT (ms): 146128.91 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.26 \nMedian TPOT (ms): 71.50 \nP99 TPOT (ms): 93.20 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.85 \nMedian ITL (ms): 62.03 \nP99 ITL (ms): 206.26 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_server.log new file mode 100644 index 0000000..c17729a --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_server.log @@ -0,0 +1,1128 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=22068) INFO 12-11 20:20:35 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=22068) INFO 12-11 20:20:35 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'trust_remote_code': True, 'max_model_len': 16384, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 32} +(APIServer pid=22068) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=22068) INFO 12-11 20:20:38 [model.py:629] Resolved architecture: Qwen3NextForCausalLM +(APIServer pid=22068) INFO 12-11 20:20:38 [model.py:1755] Using max model len 16384 +(APIServer pid=22068) INFO 12-11 20:20:38 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(APIServer pid=22068) INFO 12-11 20:20:38 [config.py:310] Disabling cascade attention since it is not supported for hybrid models. +(APIServer pid=22068) INFO 12-11 20:20:38 [config.py:437] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size. +(APIServer pid=22068) INFO 12-11 20:20:38 [config.py:461] Padding mamba page size by 1.49% to ensure that mamba page size and attention page size are exactly equal. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=22230) INFO 12-11 20:20:42 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=22230) WARNING 12-11 20:20:42 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-11 20:20:45 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:53731 backend=nccl +INFO 12-11 20:20:45 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:53731 backend=nccl +INFO 12-11 20:20:45 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-11 20:20:46 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-11 20:20:46 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=22312) INFO 12-11 20:20:46 [gpu_model_runner.py:3544] Starting to load model cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit... +(Worker_TP1 pid=22313) WARNING 12-11 20:20:47 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP1 pid=22313) INFO 12-11 20:20:47 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=22313) INFO 12-11 20:20:47 [compressed_tensors_moe.py:189] Using CompressedTensorsWNA16MoEMethod +(Worker_TP0 pid=22312) WARNING 12-11 20:20:47 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(Worker_TP0 pid=22312) INFO 12-11 20:20:47 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=22312) INFO 12-11 20:20:47 [compressed_tensors_moe.py:189] Using CompressedTensorsWNA16MoEMethod +(Worker_TP1 pid=22313) INFO 12-11 20:20:47 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=22312) INFO 12-11 20:20:47 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=22312) Loading safetensors checkpoint shards: 0% Completed | 0/10 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df39931f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.49 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 195.98 \nPeak output token throughput (tok/s): 365.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 386.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.34 \nMedian TTFT (ms): 69.07 \nP99 TTFT (ms): 180.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.10 \nMedian TPOT (ms): 37.08 \nP99 TPOT (ms): 40.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 37.10 \nMedian ITL (ms): 36.27 \nP99 ITL (ms): 58.47 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..8594ee8 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ba51e591-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 209.75 \nTotal input tokens: 145579 \nTotal generated tokens: 150640 \nRequest throughput (req/s): 3.43 \nOutput token throughput (tok/s): 718.18 \nPeak output token throughput (tok/s): 1259.00 \nPeak concurrent requests: 58.00 \nTotal Token throughput (tok/s): 1412.23 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.92 \nMedian TTFT (ms): 75.54 \nP99 TTFT (ms): 188.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 42.81 \nMedian TPOT (ms): 42.63 \nP99 TPOT (ms): 54.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 42.63 \nMedian ITL (ms): 40.26 \nP99 ITL (ms): 125.13 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..87e7888 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1043 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=3950) INFO 12-09 18:49:09 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=3950) INFO 12-09 18:49:09 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=3950) INFO 12-09 18:49:13 [model.py:629] Resolved architecture: LlamaForCausalLM +(APIServer pid=3950) INFO 12-09 18:49:13 [model.py:1755] Using max model len 65536 +(APIServer pid=3950) INFO 12-09 18:49:13 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=4113) INFO 12-09 18:49:17 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=4113) INFO 12-09 18:49:17 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:51127 backend=nccl +(EngineCore_DP0 pid=4113) INFO 12-09 18:49:17 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=4113) INFO 12-09 18:49:18 [gpu_model_runner.py:3544] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=4113) INFO 12-09 18:49:18 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=4113) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-073f047f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.90 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 203.07 \nPeak output token throughput (tok/s): 413.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 401.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.57 \nMedian TTFT (ms): 47.42 \nP99 TTFT (ms): 140.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.86 \nMedian TPOT (ms): 21.74 \nP99 TPOT (ms): 23.96 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.82 \nMedian ITL (ms): 21.46 \nP99 ITL (ms): 32.13 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json new file mode 100644 index 0000000..dd55e8d --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2ef19409-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.59 \nTotal input tokens: 145810 \nTotal generated tokens: 151842 \nRequest throughput (req/s): 3.66 \nOutput token throughput (tok/s): 772.38 \nPeak output token throughput (tok/s): 1274.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1514.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.73 \nMedian TTFT (ms): 49.35 \nP99 TTFT (ms): 135.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.29 \nMedian TPOT (ms): 24.90 \nP99 TPOT (ms): 34.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.09 \nMedian ITL (ms): 23.53 \nP99 ITL (ms): 78.49 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log new file mode 100644 index 0000000..2791aba --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log @@ -0,0 +1,1065 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=21994) INFO 12-09 20:11:08 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=21994) INFO 12-09 20:11:08 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=21994) INFO 12-09 20:11:12 [model.py:629] Resolved architecture: LlamaForCausalLM +(APIServer pid=21994) INFO 12-09 20:11:12 [model.py:1755] Using max model len 65536 +(APIServer pid=21994) INFO 12-09 20:11:12 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=22156) INFO 12-09 20:11:17 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=22156) WARNING 12-09 20:11:17 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-09 20:11:20 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:43039 backend=nccl +INFO 12-09 20:11:20 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:43039 backend=nccl +INFO 12-09 20:11:20 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-09 20:11:21 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-09 20:11:21 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=22238) INFO 12-09 20:11:21 [gpu_model_runner.py:3544] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(Worker_TP1 pid=22239) INFO 12-09 20:11:22 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=22238) INFO 12-09 20:11:22 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=22238) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84df0cbe-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 217.42 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.83 \nOutput token throughput (tok/s): 180.27 \nPeak output token throughput (tok/s): 290.00 \nPeak concurrent requests: 20.00 \nTotal Token throughput (tok/s): 358.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 129.43 \nMedian TTFT (ms): 121.63 \nP99 TTFT (ms): 431.05 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.28 \nMedian TPOT (ms): 61.83 \nP99 TPOT (ms): 74.53 \n---------------Inter-token Latency----------------\nMean ITL (ms): 61.80 \nMedian ITL (ms): 59.99 \nP99 ITL (ms): 129.94 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..509d6d9 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ea75820-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 246.91 \nTotal input tokens: 145540 \nTotal generated tokens: 152007 \nRequest throughput (req/s): 2.92 \nOutput token throughput (tok/s): 615.63 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 109.00 \nTotal Token throughput (tok/s): 1205.06 \n---------------Time to First Token----------------\nMean TTFT (ms): 4078.97 \nMedian TTFT (ms): 3857.18 \nP99 TTFT (ms): 10112.65 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 78.49 \nMedian TPOT (ms): 79.02 \nP99 TPOT (ms): 93.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 78.52 \nMedian ITL (ms): 74.18 \nP99 ITL (ms): 163.52 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..b63a645 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1050 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=9049) INFO 12-09 19:10:31 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=9049) INFO 12-09 19:10:31 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64} +(APIServer pid=9049) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=9049) INFO 12-09 19:10:35 [model.py:629] Resolved architecture: GptOssForCausalLM +(APIServer pid=9049) Parse safetensors files: 0%| | 0/3 [00:00 Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='openai/gpt-oss-20b', speculative_config=None, tokenizer='openai/gpt-oss-20b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=mxfp4, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='openai_gptoss', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=openai/gpt-oss-20b, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [parallel_state.py:1203] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://192.168.1.122:37735 backend=nccl +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [parallel_state.py:1411] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [gpu_model_runner.py:3544] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [rocm.py:320] Using Triton Attention backend on V1 engine. +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=9216) INFO 12-09 19:10:45 [mxfp4.py:171] Using Triton backend +(EngineCore_DP0 pid=9216) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7a645b9b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 199.44 \nTotal input tokens: 38756 \nTotal generated tokens: 38779 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 194.44 \nPeak output token throughput (tok/s): 395.00 \nPeak concurrent requests: 14.00 \nTotal Token throughput (tok/s): 388.76 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.40 \nMedian TTFT (ms): 67.78 \nP99 TTFT (ms): 192.76 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.60 \nMedian TPOT (ms): 28.16 \nP99 TPOT (ms): 41.84 \n---------------Inter-token Latency----------------\nMean ITL (ms): 29.20 \nMedian ITL (ms): 25.34 \nP99 ITL (ms): 71.24 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_qps4.0_latency.json b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_qps4.0_latency.json new file mode 100644 index 0000000..67a6520 --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": false, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c0304eb1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nERROR 12-09 20:35:02 [repo_utils.py:65] Error retrieving file list: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/openai/gpt-oss-20b/tree/main?recursive=True&expand=False, retrying 1 of 2\nERROR 12-09 20:35:04 [repo_utils.py:63] Error retrieving file list: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/openai/gpt-oss-20b/tree/main?recursive=True&expand=False\nERROR 12-09 20:35:04 [repo_utils.py:128] Error retrieving file list. Please ensure your `model_name_or_path``repo_type`, `token` and `revision` arguments are correctly set. Returning an empty list.\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_server.log b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_server.log new file mode 100644 index 0000000..f08bdec --- /dev/null +++ b/benchmarks/benchmark_results_amd-r9700/openai_gpt-oss-20b_tp2_server.log @@ -0,0 +1,328 @@ +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(APIServer pid=28665) INFO 12-09 20:30:09 [api_server.py:1351] vLLM API server version 0.11.2.dev690+g67475a6e8.d20251209 +(APIServer pid=28665) INFO 12-09 20:30:09 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=28665) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=28665) INFO 12-09 20:30:13 [model.py:629] Resolved architecture: GptOssForCausalLM +(APIServer pid=28665) Parse safetensors files: 0%| | 0/3 [00:00 Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +(EngineCore_DP0 pid=28832) INFO 12-09 20:30:18 [core.py:93] Initializing a V1 LLM engine (v0.11.2.dev690+g67475a6e8.d20251209) with config: model='openai/gpt-oss-20b', speculative_config=None, tokenizer='openai/gpt-oss-20b', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=mxfp4, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='openai_gptoss', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=openai/gpt-oss-20b, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False}, 'local_cache_dir': None} +(EngineCore_DP0 pid=28832) WARNING 12-09 20:30:18 [multiproc_executor.py:880] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +/opt/venv/lib64/python3.13/site-packages/torch/library.py:357: UserWarning: Warning only once for all operators, other operators may also be overridden. + Overriding a previously registered kernel for the same operator and the same dispatch key + operator: flash_attn::_flash_attn_backward(Tensor dout, Tensor q, Tensor k, Tensor v, Tensor out, Tensor softmax_lse, Tensor(a6!)? dq, Tensor(a7!)? dk, Tensor(a8!)? dv, float dropout_p, float softmax_scale, bool causal, SymInt window_size_left, SymInt window_size_right, float softcap, Tensor? alibi_slopes, bool deterministic, Tensor? rng_state=None) -> Tensor + registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 + dispatch key: ADInplaceOrView + previous kernel: no debug info + new kernel: registered at /opt/venv/lib64/python3.13/site-packages/torch/_library/custom_ops.py:926 (Triggered internally at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/core/dispatch/OperatorEntry.cpp:208.) + self.m.impl( +INFO 12-09 20:30:22 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:58745 backend=nccl +INFO 12-09 20:30:22 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:58745 backend=nccl +INFO 12-09 20:30:22 [pynccl.py:111] vLLM is using nccl==2.27.3 +INFO 12-09 20:30:22 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +INFO 12-09 20:30:22 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1 +(Worker_TP0 pid=28914) INFO 12-09 20:30:23 [gpu_model_runner.py:3544] Starting to load model openai/gpt-oss-20b... +(Worker_TP1 pid=28915) INFO 12-09 20:30:23 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP1 pid=28915) INFO 12-09 20:30:23 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP1 pid=28915) INFO 12-09 20:30:23 [mxfp4.py:171] Using Triton backend +(Worker_TP0 pid=28914) INFO 12-09 20:30:23 [rocm.py:320] Using Triton Attention backend on V1 engine. +(Worker_TP0 pid=28914) INFO 12-09 20:30:23 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(Worker_TP0 pid=28914) INFO 12-09 20:30:23 [mxfp4.py:171] Using Triton backend +(Worker_TP0 pid=28914) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00 +(APIServer pid=4288) sys.exit(main()) +(APIServer pid=4288) ^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main +(APIServer pid=4288) args.dispatch_function(args) +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd +(APIServer pid=4288) uvloop.run(run_server(args)) +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run +(APIServer pid=4288) return __asyncio.run( +(APIServer pid=4288) ^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/asyncio/runners.py", line 195, in run +(APIServer pid=4288) return runner.run(main) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/asyncio/runners.py", line 118, in run +(APIServer pid=4288) return self._loop.run_until_complete(task) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper +(APIServer pid=4288) return await main +(APIServer pid=4288) ^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1819, in run_server +(APIServer pid=4288) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1838, in run_server_worker +(APIServer pid=4288) async with build_async_engine_client( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__ +(APIServer pid=4288) return await anext(self.gen) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 183, in build_async_engine_client +(APIServer pid=4288) async with build_async_engine_client_from_engine_args( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/contextlib.py", line 210, in __aenter__ +(APIServer pid=4288) return await anext(self.gen) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 224, in build_async_engine_client_from_engine_args +(APIServer pid=4288) async_llm = AsyncLLM.from_vllm_config( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 223, in from_vllm_config +(APIServer pid=4288) return cls( +(APIServer pid=4288) ^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 114, in __init__ +(APIServer pid=4288) tokenizer = init_tokenizer_from_config(self.model_config) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 227, in init_tokenizer_from_config +(APIServer pid=4288) return get_tokenizer( +(APIServer pid=4288) ^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 191, in get_tokenizer +(APIServer pid=4288) tokenizer = TokenizerRegistry.get_tokenizer( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/registry.py", line 86, in get_tokenizer +(APIServer pid=4288) return item.from_pretrained(*args, **kwargs) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/vllm/tokenizers/hf.py", line 84, in from_pretrained +(APIServer pid=4288) tokenizer = AutoTokenizer.from_pretrained( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 1156, in from_pretrained +(APIServer pid=4288) return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2113, in from_pretrained +(APIServer pid=4288) return cls._from_pretrained( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2395, in _from_pretrained +(APIServer pid=4288) tokenizer = cls._patch_mistral_regex( +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2438, in _patch_mistral_regex +(APIServer pid=4288) if _is_local or is_base_mistral(pretrained_model_name_or_path): +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2432, in is_base_mistral +(APIServer pid=4288) model = model_info(model_id) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn +(APIServer pid=4288) return fn(*args, **kwargs) +(APIServer pid=4288) ^^^^^^^^^^^^^^^^^^^ +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/hf_api.py", line 2638, in model_info +(APIServer pid=4288) hf_raise_for_status(r) +(APIServer pid=4288) File "/venv/main/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 482, in hf_raise_for_status +(APIServer pid=4288) raise _format(HfHubHTTPError, str(e), response) from e +(APIServer pid=4288) huggingface_hub.errors.HfHubHTTPError: 503 Server Error: Service Temporarily Unavailable for url: https://huggingface.co/api/models/RedHatAI/Llama-3.1-8B-Instruct-FP8-block diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json new file mode 100644 index 0000000..1acd84f --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f177f1fc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 210.71 \nPeak output token throughput (tok/s): 430.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 411.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 145.79 \nMedian TTFT (ms): 110.27 \nP99 TTFT (ms): 399.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.46 \nMedian TPOT (ms): 22.67 \nP99 TPOT (ms): 36.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.37 \nMedian ITL (ms): 21.34 \nP99 ITL (ms): 84.18 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..de4d720 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31b9a516-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 215.08 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.35 \nOutput token throughput (tok/s): 723.67 \nPeak output token throughput (tok/s): 1208.00 \nPeak concurrent requests: 113.00 \nTotal Token throughput (tok/s): 1405.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 11329.86 \nMedian TTFT (ms): 14049.92 \nP99 TTFT (ms): 20450.77 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 40.85 \nMedian TPOT (ms): 39.82 \nP99 TPOT (ms): 83.14 \n---------------Inter-token Latency----------------\nMean ITL (ms): 40.32 \nMedian ITL (ms): 26.70 \nP99 ITL (ms): 338.67 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..96e14f0 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1027 @@ +WARNING 12-10 09:23:01 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=3017) INFO 12-10 09:23:01 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=3017) INFO 12-10 09:23:01 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 4096, 'gpu_memory_utilization': 0.86, 'max_num_seqs': 32} +(APIServer pid=3017) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=3017) INFO 12-10 09:23:09 [model.py:637] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=3017) INFO 12-10 09:23:09 [model.py:1750] Using max model len 4096 +(APIServer pid=3017) INFO 12-10 09:23:09 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=3057) INFO 12-10 09:23:18 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:36297 backend=nccl +(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=3057) INFO 12-10 09:23:19 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=3057) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-db5fc148-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.17 \nTotal input tokens: 38358 \nTotal generated tokens: 40015 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 217.27 \nPeak output token throughput (tok/s): 541.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 425.55 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.17 \nMedian TTFT (ms): 40.63 \nP99 TTFT (ms): 140.17 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 9.73 \nMedian TPOT (ms): 9.55 \nP99 TPOT (ms): 15.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 9.55 \nMedian ITL (ms): 9.45 \nP99 ITL (ms): 14.11 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..5e5dc50 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c07c837c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 189.59 \nTotal input tokens: 146694 \nTotal generated tokens: 153522 \nRequest throughput (req/s): 3.80 \nOutput token throughput (tok/s): 809.76 \nPeak output token throughput (tok/s): 1387.00 \nPeak concurrent requests: 40.00 \nTotal Token throughput (tok/s): 1583.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 83.62 \nMedian TTFT (ms): 71.44 \nP99 TTFT (ms): 196.63 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.34 \nMedian TPOT (ms): 21.97 \nP99 TPOT (ms): 35.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.92 \nMedian ITL (ms): 19.31 \nP99 ITL (ms): 107.58 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..fc8e21f --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1026 @@ +WARNING 12-10 09:44:20 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=3923) INFO 12-10 09:44:20 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=3923) INFO 12-10 09:44:20 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} +(APIServer pid=3923) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=3923) INFO 12-10 09:44:29 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=3923) INFO 12-10 09:44:29 [model.py:1750] Using max model len 24576 +(APIServer pid=3923) INFO 12-10 09:44:29 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:37 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:49511 backend=nccl +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:38 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=3963) INFO 12-10 09:44:39 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod +(EngineCore_DP0 pid=3963) WARNING 12-10 09:44:39 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=3963) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bec359dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.18 \nTotal input tokens: 37841 \nTotal generated tokens: 38760 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 202.74 \nPeak output token throughput (tok/s): 418.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 400.67 \n---------------Time to First Token----------------\nMean TTFT (ms): 95.72 \nMedian TTFT (ms): 55.76 \nP99 TTFT (ms): 257.24 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.29 \nMedian TPOT (ms): 23.04 \nP99 TPOT (ms): 28.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.29 \nMedian ITL (ms): 22.01 \nP99 ITL (ms): 67.09 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..903dc61 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-acbcf111-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 199.30 \nTotal input tokens: 145810 \nTotal generated tokens: 151171 \nRequest throughput (req/s): 3.61 \nOutput token throughput (tok/s): 758.52 \nPeak output token throughput (tok/s): 1326.00 \nPeak concurrent requests: 52.00 \nTotal Token throughput (tok/s): 1490.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 134.75 \nMedian TTFT (ms): 130.98 \nP99 TTFT (ms): 409.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 36.74 \nMedian TPOT (ms): 35.30 \nP99 TPOT (ms): 77.70 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.30 \nMedian ITL (ms): 26.57 \nP99 ITL (ms): 192.53 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..1a11313 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1023 @@ +WARNING 12-10 10:12:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=4826) INFO 12-10 10:12:27 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=4826) INFO 12-10 10:12:27 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 31800, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=4826) INFO 12-10 10:12:36 [model.py:637] Resolved architecture: LlamaForCausalLM +(APIServer pid=4826) INFO 12-10 10:12:36 [model.py:1750] Using max model len 31800 +(APIServer pid=4826) INFO 12-10 10:12:36 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=31800, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:45209 backend=nccl +(EngineCore_DP0 pid=4866) INFO 12-10 10:12:46 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=4866) INFO 12-10 10:12:47 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=4866) INFO 12-10 10:12:47 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=4866) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b23a48ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.20 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.78 \nPeak output token throughput (tok/s): 495.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 423.18 \n---------------Time to First Token----------------\nMean TTFT (ms): 60.23 \nMedian TTFT (ms): 37.85 \nP99 TTFT (ms): 146.84 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.21 \nMedian TPOT (ms): 11.26 \nP99 TPOT (ms): 18.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.00 \nMedian ITL (ms): 10.95 \nP99 ITL (ms): 19.93 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..4ea3aa8 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2c7096a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.29 \nTotal input tokens: 145540 \nTotal generated tokens: 151404 \nRequest throughput (req/s): 3.78 \nOutput token throughput (tok/s): 795.67 \nPeak output token throughput (tok/s): 1315.00 \nPeak concurrent requests: 36.00 \nTotal Token throughput (tok/s): 1560.51 \n---------------Time to First Token----------------\nMean TTFT (ms): 65.82 \nMedian TTFT (ms): 45.53 \nP99 TTFT (ms): 177.98 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 20.36 \nMedian TPOT (ms): 20.02 \nP99 TPOT (ms): 31.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 20.10 \nMedian ITL (ms): 18.54 \nP99 ITL (ms): 90.23 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..7036a6f --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-3090/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1022 @@ +WARNING 12-10 08:45:12 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=1916) INFO 12-10 08:45:12 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=1916) INFO 12-10 08:45:12 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 16384, 'max_num_seqs': 32} +(APIServer pid=1916) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=1916) INFO 12-10 08:45:19 [model.py:637] Resolved architecture: GptOssForCausalLM +(APIServer pid=1916) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:59799 backend=nccl +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:30 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=1960) INFO 12-10 08:45:31 [mxfp4.py:162] Using Marlin backend +(EngineCore_DP0 pid=1960) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96888f07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.82 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 211.17 \nPeak output token throughput (tok/s): 443.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 412.18 \n---------------Time to First Token----------------\nMean TTFT (ms): 64.41 \nMedian TTFT (ms): 50.82 \nP99 TTFT (ms): 144.55 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.14 \nMedian TPOT (ms): 20.98 \nP99 TPOT (ms): 23.59 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.12 \nMedian ITL (ms): 20.65 \nP99 ITL (ms): 29.23 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..d623116 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-bd32e896-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 194.55 \nTotal input tokens: 146694 \nTotal generated tokens: 155576 \nRequest throughput (req/s): 3.70 \nOutput token throughput (tok/s): 799.66 \nPeak output token throughput (tok/s): 1300.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1553.66 \n---------------Time to First Token----------------\nMean TTFT (ms): 92.77 \nMedian TTFT (ms): 68.07 \nP99 TTFT (ms): 484.98 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.78 \nMedian TPOT (ms): 25.52 \nP99 TPOT (ms): 34.37 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.65 \nMedian ITL (ms): 23.39 \nP99 ITL (ms): 86.03 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..49fed97 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1024 @@ +WARNING 12-09 21:03:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=31169) INFO 12-09 21:03:27 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=31169) INFO 12-09 21:03:27 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 4096, 'gpu_memory_utilization': 0.86, 'max_num_seqs': 32} +(APIServer pid=31169) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=31169) INFO 12-09 21:03:34 [model.py:637] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=31169) INFO 12-09 21:03:34 [model.py:1750] Using max model len 4096 +(APIServer pid=31169) INFO 12-09 21:03:34 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=31480) INFO 12-09 21:03:41 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:34731 backend=nccl +(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=31480) INFO 12-09 21:03:42 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=31480) INFO 12-09 21:03:43 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=31480) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4cdb5c8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.35 \nTotal input tokens: 38358 \nTotal generated tokens: 39718 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 216.63 \nPeak output token throughput (tok/s): 666.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 425.84 \n---------------Time to First Token----------------\nMean TTFT (ms): 37.48 \nMedian TTFT (ms): 30.33 \nP99 TTFT (ms): 77.17 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 7.36 \nMedian TPOT (ms): 7.11 \nP99 TPOT (ms): 11.45 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.26 \nMedian ITL (ms): 6.53 \nP99 ITL (ms): 11.05 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..0211464 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-81c0edc1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 186.20 \nTotal input tokens: 146694 \nTotal generated tokens: 153033 \nRequest throughput (req/s): 3.87 \nOutput token throughput (tok/s): 821.89 \nPeak output token throughput (tok/s): 1471.00 \nPeak concurrent requests: 30.00 \nTotal Token throughput (tok/s): 1609.74 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.36 \nMedian TTFT (ms): 42.14 \nP99 TTFT (ms): 82.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 13.89 \nMedian TPOT (ms): 13.91 \nP99 TPOT (ms): 17.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 13.74 \nMedian ITL (ms): 13.29 \nP99 ITL (ms): 38.23 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..29c36f8 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1026 @@ +WARNING 12-09 21:18:19 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=35981) INFO 12-09 21:18:19 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=35981) INFO 12-09 21:18:19 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} +(APIServer pid=35981) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=35981) INFO 12-09 21:18:25 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=35981) INFO 12-09 21:18:25 [model.py:1750] Using max model len 24576 +(APIServer pid=35981) INFO 12-09 21:18:26 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:33 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:58697 backend=nccl +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:34 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=36287) INFO 12-09 21:18:35 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod +(EngineCore_DP0 pid=36287) WARNING 12-09 21:18:35 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=36287) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-04789567-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 188.87 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 205.96 \nPeak output token throughput (tok/s): 465.00 \nPeak concurrent requests: 10.00 \nTotal Token throughput (tok/s): 406.32 \n---------------Time to First Token----------------\nMean TTFT (ms): 52.16 \nMedian TTFT (ms): 44.25 \nP99 TTFT (ms): 112.79 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 17.68 \nMedian TPOT (ms): 17.62 \nP99 TPOT (ms): 19.35 \n---------------Inter-token Latency----------------\nMean ITL (ms): 17.64 \nMedian ITL (ms): 17.35 \nP99 ITL (ms): 23.66 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..a8b55d3 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b098e8e8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 193.10 \nTotal input tokens: 145810 \nTotal generated tokens: 152091 \nRequest throughput (req/s): 3.73 \nOutput token throughput (tok/s): 787.62 \nPeak output token throughput (tok/s): 1351.00 \nPeak concurrent requests: 39.00 \nTotal Token throughput (tok/s): 1542.71 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.32 \nMedian TTFT (ms): 49.20 \nP99 TTFT (ms): 126.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 20.27 \nMedian TPOT (ms): 20.07 \nP99 TPOT (ms): 27.95 \n---------------Inter-token Latency----------------\nMean ITL (ms): 20.15 \nMedian ITL (ms): 19.30 \nP99 ITL (ms): 62.76 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..8e6a030 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1023 @@ +WARNING 12-09 19:47:08 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=5841) INFO 12-09 19:47:08 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=5841) INFO 12-09 19:47:08 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 31800, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=5841) INFO 12-09 19:47:17 [model.py:637] Resolved architecture: LlamaForCausalLM +(APIServer pid=5841) INFO 12-09 19:47:17 [model.py:1750] Using max model len 31800 +(APIServer pid=5841) INFO 12-09 19:47:17 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=31800, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:35389 backend=nccl +(EngineCore_DP0 pid=6154) INFO 12-09 19:47:26 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=6154) INFO 12-09 19:47:27 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=6154) INFO 12-09 19:47:27 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=6154) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-31ba87dc-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.12 \nTotal input tokens: 38756 \nTotal generated tokens: 38869 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.27 \nPeak output token throughput (tok/s): 609.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 423.91 \n---------------Time to First Token----------------\nMean TTFT (ms): 40.22 \nMedian TTFT (ms): 36.83 \nP99 TTFT (ms): 74.88 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.07 \nMedian TPOT (ms): 8.23 \nP99 TPOT (ms): 11.78 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.04 \nMedian ITL (ms): 8.17 \nP99 ITL (ms): 12.73 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..7a9f6b1 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b94205ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 187.07 \nTotal input tokens: 145540 \nTotal generated tokens: 151811 \nRequest throughput (req/s): 3.85 \nOutput token throughput (tok/s): 811.50 \nPeak output token throughput (tok/s): 1549.00 \nPeak concurrent requests: 30.00 \nTotal Token throughput (tok/s): 1589.49 \n---------------Time to First Token----------------\nMean TTFT (ms): 41.02 \nMedian TTFT (ms): 36.91 \nP99 TTFT (ms): 79.84 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.71 \nMedian TPOT (ms): 14.74 \nP99 TPOT (ms): 17.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.60 \nMedian ITL (ms): 14.44 \nP99 ITL (ms): 32.49 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..179a843 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-4090/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1022 @@ +WARNING 12-09 20:20:45 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=19409) INFO 12-09 20:20:45 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=19409) INFO 12-09 20:20:45 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 16384, 'max_num_seqs': 32} +(APIServer pid=19409) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=19409) INFO 12-09 20:20:53 [model.py:637] Resolved architecture: GptOssForCausalLM +(APIServer pid=19409) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:02 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:56281 backend=nccl +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:02 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=19724) INFO 12-09 20:21:03 [mxfp4.py:162] Using Marlin backend +(EngineCore_DP0 pid=19724) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0abaed61-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.06 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 209.81 \nPeak output token throughput (tok/s): 440.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 409.54 \n---------------Time to First Token----------------\nMean TTFT (ms): 51.40 \nMedian TTFT (ms): 47.89 \nP99 TTFT (ms): 93.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 23.06 \nMedian TPOT (ms): 23.00 \nP99 TPOT (ms): 24.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 23.10 \nMedian ITL (ms): 22.83 \nP99 ITL (ms): 28.65 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..96eddac --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-14719fc9-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 195.24 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.69 \nOutput token throughput (tok/s): 796.89 \nPeak output token throughput (tok/s): 1321.00 \nPeak concurrent requests: 39.00 \nTotal Token throughput (tok/s): 1548.24 \n---------------Time to First Token----------------\nMean TTFT (ms): 53.18 \nMedian TTFT (ms): 49.55 \nP99 TTFT (ms): 101.04 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 24.41 \nMedian TPOT (ms): 24.26 \nP99 TPOT (ms): 28.40 \n---------------Inter-token Latency----------------\nMean ITL (ms): 24.37 \nMedian ITL (ms): 23.61 \nP99 ITL (ms): 56.70 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..64143d2 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1032 @@ +WARNING 12-09 19:04:10 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=14033) INFO 12-09 19:04:10 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=14033) INFO 12-09 19:04:10 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} +(APIServer pid=14033) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=14033) INFO 12-09 19:04:16 [model.py:637] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=14033) INFO 12-09 19:04:16 [model.py:1750] Using max model len 32768 +(APIServer pid=14033) INFO 12-09 19:04:17 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=14297) INFO 12-09 19:04:23 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:39463 backend=nccl +(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=14297) INFO 12-09 19:04:24 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=14297) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5741d2ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 182.76 \nTotal input tokens: 38358 \nTotal generated tokens: 39611 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 216.74 \nPeak output token throughput (tok/s): 789.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 426.61 \n---------------Time to First Token----------------\nMean TTFT (ms): 30.25 \nMedian TTFT (ms): 23.16 \nP99 TTFT (ms): 57.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 5.38 \nMedian TPOT (ms): 5.32 \nP99 TPOT (ms): 7.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 5.36 \nMedian ITL (ms): 5.29 \nP99 ITL (ms): 7.43 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..ab807da --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4730dd07-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 183.59 \nTotal input tokens: 146694 \nTotal generated tokens: 152861 \nRequest throughput (req/s): 3.92 \nOutput token throughput (tok/s): 832.64 \nPeak output token throughput (tok/s): 1713.00 \nPeak concurrent requests: 25.00 \nTotal Token throughput (tok/s): 1631.69 \n---------------Time to First Token----------------\nMean TTFT (ms): 29.84 \nMedian TTFT (ms): 24.09 \nP99 TTFT (ms): 60.04 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.01 \nMedian TPOT (ms): 7.77 \nP99 TPOT (ms): 11.35 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.96 \nMedian ITL (ms): 7.28 \nP99 ITL (ms): 27.43 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..5575e22 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1029 @@ +WARNING 12-09 19:23:27 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=17193) INFO 12-09 19:23:27 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=17193) INFO 12-09 19:23:27 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} +(APIServer pid=17193) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=17193) INFO 12-09 19:23:33 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=17193) INFO 12-09 19:23:33 [model.py:1750] Using max model len 24576 +(APIServer pid=17193) INFO 12-09 19:23:35 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:41 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:60659 backend=nccl +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:42 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=17457) INFO 12-09 19:23:43 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod +(EngineCore_DP0 pid=17457) WARNING 12-09 19:23:43 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=17457) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d6b41380-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 185.59 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 209.60 \nPeak output token throughput (tok/s): 517.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 413.50 \n---------------Time to First Token----------------\nMean TTFT (ms): 39.73 \nMedian TTFT (ms): 32.58 \nP99 TTFT (ms): 92.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.35 \nMedian TPOT (ms): 11.23 \nP99 TPOT (ms): 13.06 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.32 \nMedian ITL (ms): 11.05 \nP99 ITL (ms): 13.41 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..aa4f809 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-10085502-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 188.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152175 \nRequest throughput (req/s): 3.82 \nOutput token throughput (tok/s): 807.79 \nPeak output token throughput (tok/s): 1510.00 \nPeak concurrent requests: 29.00 \nTotal Token throughput (tok/s): 1581.79 \n---------------Time to First Token----------------\nMean TTFT (ms): 38.75 \nMedian TTFT (ms): 31.18 \nP99 TTFT (ms): 92.33 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 12.77 \nMedian TPOT (ms): 12.43 \nP99 TPOT (ms): 17.06 \n---------------Inter-token Latency----------------\nMean ITL (ms): 12.71 \nMedian ITL (ms): 11.91 \nP99 ITL (ms): 36.70 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..2a23afe --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1027 @@ +WARNING 12-09 18:18:16 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=4406) INFO 12-09 18:18:16 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=4406) INFO 12-09 18:18:16 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=4406) INFO 12-09 18:18:23 [model.py:637] Resolved architecture: LlamaForCausalLM +(APIServer pid=4406) INFO 12-09 18:18:23 [model.py:1750] Using max model len 65536 +(APIServer pid=4406) INFO 12-09 18:18:24 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:34945 backend=nccl +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:31 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:32 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:32 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=4670) INFO 12-09 18:18:34 [weight_utils.py:487] Time spent downloading weights for meta-llama/Meta-Llama-3.1-8B-Instruct: 0.680210 seconds +(EngineCore_DP0 pid=4670) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-7ca49d8d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 182.22 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.99 \nOutput token throughput (tok/s): 215.09 \nPeak output token throughput (tok/s): 788.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 427.78 \n---------------Time to First Token----------------\nMean TTFT (ms): 26.17 \nMedian TTFT (ms): 21.35 \nP99 TTFT (ms): 52.39 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 5.08 \nMedian TPOT (ms): 5.07 \nP99 TPOT (ms): 7.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 5.10 \nMedian ITL (ms): 5.12 \nP99 ITL (ms): 7.77 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..81bccdf --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1904b29f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 183.65 \nTotal input tokens: 145540 \nTotal generated tokens: 151340 \nRequest throughput (req/s): 3.92 \nOutput token throughput (tok/s): 824.06 \nPeak output token throughput (tok/s): 1801.00 \nPeak concurrent requests: 25.00 \nTotal Token throughput (tok/s): 1616.54 \n---------------Time to First Token----------------\nMean TTFT (ms): 26.49 \nMedian TTFT (ms): 22.28 \nP99 TTFT (ms): 56.70 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.24 \nMedian TPOT (ms): 8.09 \nP99 TPOT (ms): 10.51 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.19 \nMedian ITL (ms): 7.72 \nP99 ITL (ms): 14.51 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..e415e3d --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-5090/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1030 @@ +WARNING 12-09 18:35:12 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=8738) INFO 12-09 18:35:12 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=8738) INFO 12-09 18:35:12 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=8738) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=8738) INFO 12-09 18:35:19 [model.py:637] Resolved architecture: GptOssForCausalLM +(APIServer pid=8738) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.4:35797 backend=nccl +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=9006) INFO 12-09 18:35:28 [mxfp4.py:162] Using Marlin backend +(EngineCore_DP0 pid=9006) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-66e0ceca-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 187.88 \nTotal input tokens: 38358 \nTotal generated tokens: 40286 \nRequest throughput (req/s): 0.96 \nOutput token throughput (tok/s): 214.42 \nPeak output token throughput (tok/s): 460.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 418.59 \n---------------Time to First Token----------------\nMean TTFT (ms): 77.11 \nMedian TTFT (ms): 52.45 \nP99 TTFT (ms): 199.22 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 15.76 \nMedian TPOT (ms): 15.43 \nP99 TPOT (ms): 21.30 \n---------------Inter-token Latency----------------\nMean ITL (ms): 15.60 \nMedian ITL (ms): 14.96 \nP99 ITL (ms): 17.95 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..ff2975f --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-702c15c4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.21 \nTotal input tokens: 146694 \nTotal generated tokens: 155632 \nRequest throughput (req/s): 3.79 \nOutput token throughput (tok/s): 818.19 \nPeak output token throughput (tok/s): 1520.00 \nPeak concurrent requests: 37.00 \nTotal Token throughput (tok/s): 1589.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 76.15 \nMedian TTFT (ms): 45.58 \nP99 TTFT (ms): 247.34 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.54 \nMedian TPOT (ms): 18.83 \nP99 TPOT (ms): 35.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.12 \nMedian ITL (ms): 16.70 \nP99 ITL (ms): 114.66 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..8e23e99 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1024 @@ +WARNING 12-10 09:47:44 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=6553) INFO 12-10 09:47:44 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=6553) INFO 12-10 09:47:44 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} +(APIServer pid=6553) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=6553) INFO 12-10 09:47:54 [model.py:637] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=6553) INFO 12-10 09:47:54 [model.py:1750] Using max model len 32768 +(APIServer pid=6553) INFO 12-10 09:47:54 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:36973 backend=nccl +(EngineCore_DP0 pid=6815) INFO 12-10 09:48:03 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=6815) INFO 12-10 09:48:04 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=6815) INFO 12-10 09:48:05 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=6815) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b80b1bb5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.14 \nTotal input tokens: 38358 \nTotal generated tokens: 39211 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 212.94 \nPeak output token throughput (tok/s): 560.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 421.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.12 \nMedian TTFT (ms): 41.56 \nP99 TTFT (ms): 82.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 8.89 \nMedian TPOT (ms): 8.73 \nP99 TPOT (ms): 13.87 \n---------------Inter-token Latency----------------\nMean ITL (ms): 8.70 \nMedian ITL (ms): 8.44 \nP99 ITL (ms): 11.78 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..6f2703c --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fb44348e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 186.80 \nTotal input tokens: 146694 \nTotal generated tokens: 153333 \nRequest throughput (req/s): 3.85 \nOutput token throughput (tok/s): 820.85 \nPeak output token throughput (tok/s): 1449.00 \nPeak concurrent requests: 31.00 \nTotal Token throughput (tok/s): 1606.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.68 \nMedian TTFT (ms): 36.77 \nP99 TTFT (ms): 89.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.46 \nMedian TPOT (ms): 14.20 \nP99 TPOT (ms): 19.40 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.28 \nMedian ITL (ms): 13.63 \nP99 ITL (ms): 44.44 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..8c135cc --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1026 @@ +WARNING 12-10 10:05:09 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=8737) INFO 12-10 10:05:09 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=8737) INFO 12-10 10:05:09 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} +(APIServer pid=8737) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=8737) INFO 12-10 10:05:19 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=8737) INFO 12-10 10:05:19 [model.py:1750] Using max model len 24576 +(APIServer pid=8737) INFO 12-10 10:05:19 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:28 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:47671 backend=nccl +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:29 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=8999) INFO 12-10 10:05:30 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod +(EngineCore_DP0 pid=8999) WARNING 12-10 10:05:30 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=8999) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4225f4cd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 187.19 \nTotal input tokens: 37841 \nTotal generated tokens: 38830 \nRequest throughput (req/s): 0.96 \nOutput token throughput (tok/s): 207.43 \nPeak output token throughput (tok/s): 490.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 409.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 45.84 \nMedian TTFT (ms): 37.87 \nP99 TTFT (ms): 101.24 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.35 \nMedian TPOT (ms): 14.27 \nP99 TPOT (ms): 16.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.28 \nMedian ITL (ms): 14.04 \nP99 ITL (ms): 16.81 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..5346632 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-4ca67e7d-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 190.55 \nTotal input tokens: 145810 \nTotal generated tokens: 152171 \nRequest throughput (req/s): 3.78 \nOutput token throughput (tok/s): 798.60 \nPeak output token throughput (tok/s): 1492.00 \nPeak concurrent requests: 35.00 \nTotal Token throughput (tok/s): 1563.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 44.19 \nMedian TTFT (ms): 37.54 \nP99 TTFT (ms): 101.54 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 16.05 \nMedian TPOT (ms): 15.69 \nP99 TPOT (ms): 20.99 \n---------------Inter-token Latency----------------\nMean ITL (ms): 15.95 \nMedian ITL (ms): 15.02 \nP99 ITL (ms): 41.81 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..354c6bc --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1022 @@ +WARNING 12-10 09:24:11 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=2740) INFO 12-10 09:24:11 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=2740) INFO 12-10 09:24:11 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=2740) INFO 12-10 09:24:21 [model.py:637] Resolved architecture: LlamaForCausalLM +(APIServer pid=2740) INFO 12-10 09:24:21 [model.py:1750] Using max model len 65536 +(APIServer pid=2740) INFO 12-10 09:24:21 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:46573 backend=nccl +(EngineCore_DP0 pid=3002) INFO 12-10 09:24:30 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=3002) INFO 12-10 09:24:31 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=3002) INFO 12-10 09:24:32 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=3002) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-85d6e157-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 183.05 \nTotal input tokens: 38756 \nTotal generated tokens: 38779 \nRequest throughput (req/s): 0.98 \nOutput token throughput (tok/s): 211.85 \nPeak output token throughput (tok/s): 676.00 \nPeak concurrent requests: 8.00 \nTotal Token throughput (tok/s): 423.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 36.19 \nMedian TTFT (ms): 30.20 \nP99 TTFT (ms): 83.37 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 7.40 \nMedian TPOT (ms): 7.32 \nP99 TPOT (ms): 10.60 \n---------------Inter-token Latency----------------\nMean ITL (ms): 7.34 \nMedian ITL (ms): 7.07 \nP99 ITL (ms): 10.69 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..f8b7077 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-584a872e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 185.93 \nTotal input tokens: 145540 \nTotal generated tokens: 151690 \nRequest throughput (req/s): 3.87 \nOutput token throughput (tok/s): 815.86 \nPeak output token throughput (tok/s): 1630.00 \nPeak concurrent requests: 28.00 \nTotal Token throughput (tok/s): 1598.63 \n---------------Time to First Token----------------\nMean TTFT (ms): 38.36 \nMedian TTFT (ms): 31.91 \nP99 TTFT (ms): 78.67 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 12.67 \nMedian TPOT (ms): 12.82 \nP99 TPOT (ms): 16.44 \n---------------Inter-token Latency----------------\nMean ITL (ms): 12.51 \nMedian ITL (ms): 12.62 \nP99 ITL (ms): 31.79 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..fb0eaef --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-a100/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1023 @@ +WARNING 12-10 10:51:19 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=15750) INFO 12-10 10:51:19 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=15750) INFO 12-10 10:51:19 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 24000, 'max_num_seqs': 64} +(APIServer pid=15750) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=15750) INFO 12-10 10:51:29 [model.py:637] Resolved architecture: GptOssForCausalLM +(APIServer pid=15750) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:40 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.2:59277 backend=nccl +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:40 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=16016) INFO 12-10 10:51:41 [mxfp4.py:162] Using Marlin backend +(EngineCore_DP0 pid=16016) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-39a9e102-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 197.48 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 204.06 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 398.30 \n---------------Time to First Token----------------\nMean TTFT (ms): 78.01 \nMedian TTFT (ms): 70.55 \nP99 TTFT (ms): 149.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 34.15 \nMedian TPOT (ms): 34.06 \nP99 TPOT (ms): 36.67 \n---------------Inter-token Latency----------------\nMean ITL (ms): 34.16 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 58.95 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json new file mode 100644 index 0000000..1b3d37f --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0e099965-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 205.53 \nTotal input tokens: 146694 \nTotal generated tokens: 155647 \nRequest throughput (req/s): 3.50 \nOutput token throughput (tok/s): 757.31 \nPeak output token throughput (tok/s): 1260.00 \nPeak concurrent requests: 59.00 \nTotal Token throughput (tok/s): 1471.06 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.03 \nMedian TTFT (ms): 76.62 \nP99 TTFT (ms): 176.99 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.66 \nMedian TPOT (ms): 41.32 \nP99 TPOT (ms): 52.93 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.47 \nMedian ITL (ms): 39.19 \nP99 ITL (ms): 111.74 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log new file mode 100644 index 0000000..d2caafb --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log @@ -0,0 +1,1025 @@ +WARNING 12-10 19:52:40 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=4723) INFO 12-10 19:52:40 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=4723) INFO 12-10 19:52:40 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32768, 'max_num_seqs': 64} +(APIServer pid=4723) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=4723) INFO 12-10 19:52:45 [model.py:637] Resolved architecture: Qwen3ForCausalLM +(APIServer pid=4723) INFO 12-10 19:52:45 [model.py:1750] Using max model len 32768 +(APIServer pid=4723) INFO 12-10 19:52:45 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=4827) INFO 12-10 19:52:49 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32768, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:34143 backend=nccl +(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=4827) INFO 12-10 19:52:50 [gpu_model_runner.py:3467] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic... +(EngineCore_DP0 pid=4827) INFO 12-10 19:52:51 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=4827) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-341eb0ef-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 184.76 \nTotal input tokens: 38358 \nTotal generated tokens: 40087 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 216.97 \nPeak output token throughput (tok/s): 453.00 \nPeak concurrent requests: 9.00 \nTotal Token throughput (tok/s): 424.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 40.37 \nMedian TTFT (ms): 37.70 \nP99 TTFT (ms): 70.13 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 11.50 \nMedian TPOT (ms): 11.56 \nP99 TPOT (ms): 16.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 11.24 \nMedian ITL (ms): 11.23 \nP99 ITL (ms): 19.54 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json new file mode 100644 index 0000000..be18451 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-34321bd4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 192.26 \nTotal input tokens: 146694 \nTotal generated tokens: 153536 \nRequest throughput (req/s): 3.74 \nOutput token throughput (tok/s): 798.60 \nPeak output token throughput (tok/s): 1194.00 \nPeak concurrent requests: 46.00 \nTotal Token throughput (tok/s): 1561.61 \n---------------Time to First Token----------------\nMean TTFT (ms): 54.49 \nMedian TTFT (ms): 52.17 \nP99 TTFT (ms): 97.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.72 \nMedian TPOT (ms): 25.55 \nP99 TPOT (ms): 31.80 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.55 \nMedian ITL (ms): 24.86 \nP99 ITL (ms): 53.01 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log new file mode 100644 index 0000000..54d348e --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log @@ -0,0 +1,1025 @@ +WARNING 12-10 20:10:10 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=5918) INFO 12-10 20:10:10 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=5918) INFO 12-10 20:10:10 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'max_num_seqs': 64} +(APIServer pid=5918) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=5918) INFO 12-10 20:10:17 [model.py:637] Resolved architecture: Qwen3MoeForCausalLM +(APIServer pid=5918) INFO 12-10 20:10:17 [model.py:1750] Using max model len 24576 +(APIServer pid=5918) INFO 12-10 20:10:18 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:55455 backend=nccl +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:23 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [gpu_model_runner.py:3467] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit... +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [compressed_tensors_wNa16.py:114] Using MarlinLinearKernel for CompressedTensorsWNA16 +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=6022) INFO 12-10 20:10:24 [compressed_tensors_moe.py:167] Using CompressedTensorsWNA16MarlinMoEMethod +(EngineCore_DP0 pid=6022) WARNING 12-10 20:10:24 [compressed_tensors.py:721] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +(EngineCore_DP0 pid=6022) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f41559c3-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.60 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.88 \nPeak output token throughput (tok/s): 395.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.34 \n---------------Time to First Token----------------\nMean TTFT (ms): 63.33 \nMedian TTFT (ms): 59.42 \nP99 TTFT (ms): 123.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.79 \nMedian TPOT (ms): 30.76 \nP99 TPOT (ms): 32.51 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.78 \nMedian ITL (ms): 30.39 \nP99 ITL (ms): 42.55 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json new file mode 100644 index 0000000..dd7925e --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f456f8a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.29 \nTotal input tokens: 145785 \nTotal generated tokens: 151865 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 743.36 \nPeak output token throughput (tok/s): 1197.00 \nPeak concurrent requests: 52.00 \nTotal Token throughput (tok/s): 1456.96 \n---------------Time to First Token----------------\nMean TTFT (ms): 68.23 \nMedian TTFT (ms): 63.70 \nP99 TTFT (ms): 136.63 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 34.84 \nMedian TPOT (ms): 34.66 \nP99 TPOT (ms): 41.64 \n---------------Inter-token Latency----------------\nMean ITL (ms): 34.70 \nMedian ITL (ms): 33.44 \nP99 ITL (ms): 79.44 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log new file mode 100644 index 0000000..c84c39b --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log @@ -0,0 +1,1021 @@ +WARNING 12-10 19:17:54 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=1907) INFO 12-10 19:17:54 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=1907) INFO 12-10 19:17:54 [utils.py:253] non-default args: {'model_tag': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'host': '127.0.0.1', 'model': 'meta-llama/Meta-Llama-3.1-8B-Instruct', 'max_model_len': 65536, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=1907) INFO 12-10 19:18:00 [model.py:637] Resolved architecture: LlamaForCausalLM +(APIServer pid=1907) INFO 12-10 19:18:00 [model.py:1750] Using max model len 65536 +(APIServer pid=1907) INFO 12-10 19:18:00 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048. +(EngineCore_DP0 pid=2011) INFO 12-10 19:18:05 [core.py:93] Initializing a V1 LLM engine (v0.12.0) with config: model='meta-llama/Meta-Llama-3.1-8B-Instruct', speculative_config=None, tokenizer='meta-llama/Meta-Llama-3.1-8B-Instruct', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=65536, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01), seed=0, served_model_name=meta-llama/Meta-Llama-3.1-8B-Instruct, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:45173 backend=nccl +(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=2011) INFO 12-10 19:18:06 [gpu_model_runner.py:3467] Starting to load model meta-llama/Meta-Llama-3.1-8B-Instruct... +(EngineCore_DP0 pid=2011) INFO 12-10 19:18:07 [cuda.py:411] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'] +(EngineCore_DP0 pid=2011) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-68841ca7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 185.44 \nTotal input tokens: 38756 \nTotal generated tokens: 38777 \nRequest throughput (req/s): 0.97 \nOutput token throughput (tok/s): 209.11 \nPeak output token throughput (tok/s): 437.00 \nPeak concurrent requests: 10.00 \nTotal Token throughput (tok/s): 418.10 \n---------------Time to First Token----------------\nMean TTFT (ms): 42.77 \nMedian TTFT (ms): 40.02 \nP99 TTFT (ms): 73.37 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 14.89 \nMedian TPOT (ms): 14.86 \nP99 TPOT (ms): 21.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 14.57 \nMedian ITL (ms): 14.54 \nP99 ITL (ms): 24.86 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json new file mode 100644 index 0000000..0c20aeb --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_qps4.0_latency.json @@ -0,0 +1,4 @@ +{ + "success": true, + "raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-96562708-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, tokenizer_mode='auto', served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 194.32 \nTotal input tokens: 145540 \nTotal generated tokens: 151349 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 778.86 \nPeak output token throughput (tok/s): 1218.00 \nPeak concurrent requests: 42.00 \nTotal Token throughput (tok/s): 1527.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 51.01 \nMedian TTFT (ms): 49.88 \nP99 TTFT (ms): 83.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 26.15 \nMedian TPOT (ms): 26.20 \nP99 TPOT (ms): 28.93 \n---------------Inter-token Latency----------------\nMean ITL (ms): 26.04 \nMedian ITL (ms): 25.85 \nP99 ITL (ms): 42.96 \n==================================================\n" +} \ No newline at end of file diff --git a/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log new file mode 100644 index 0000000..5ee2883 --- /dev/null +++ b/benchmarks/benchmark_results_nvidia-ada5000/openai_gpt-oss-20b_tp1_server.log @@ -0,0 +1,1022 @@ +WARNING 12-10 19:34:31 [argparse_utils.py:195] With `vllm serve`, you should provide the model as a positional argument or in a config file instead of via the `--model` option. The `--model` option will be removed in v0.13. +(APIServer pid=3207) INFO 12-10 19:34:31 [api_server.py:1772] vLLM API server version 0.12.0 +(APIServer pid=3207) INFO 12-10 19:34:31 [utils.py:253] non-default args: {'model_tag': 'openai/gpt-oss-20b', 'host': '127.0.0.1', 'model': 'openai/gpt-oss-20b', 'trust_remote_code': True, 'max_model_len': 32768, 'gpu_memory_utilization': 0.95, 'max_num_seqs': 64} +(APIServer pid=3207) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored. +(APIServer pid=3207) INFO 12-10 19:34:38 [model.py:637] Resolved architecture: GptOssForCausalLM +(APIServer pid=3207) Parse safetensors files: 0%| | 0/3 [00:00, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, 992, 1008, 1024], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 1024, 'dynamic_shapes_config': {'type': }, 'local_cache_dir': None} +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:48 [parallel_state.py:1200] world_size=1 rank=0 local_rank=0 distributed_init_method=tcp://172.17.0.3:45235 backend=nccl +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:48 [parallel_state.py:1408] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0 +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [gpu_model_runner.py:3467] Starting to load model openai/gpt-oss-20b... +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [cuda.py:411] Using TRITON_ATTN attention backend out of potential backends: ['TRITON_ATTN'] +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [layer.py:379] Enabled separate cuda stream for MoE shared_experts +(EngineCore_DP0 pid=3315) INFO 12-10 19:34:49 [mxfp4.py:162] Using Marlin backend +(EngineCore_DP0 pid=3315) Loading safetensors checkpoint shards: 0% Completed | 0/3 [00:00 timeout: + log("CRITICAL: Cleanup timed out! Force attempting `killall -9 vllm` as last resort.") + subprocess.run("killall -9 vllm", shell=True, stderr=subprocess.DEVNULL) + break + + time.sleep(1.5) # Wait a bit before hammering again + + +def wait_for_server_and_parse(process, timeout=300): + """ + Waits for server to be ready. + Parses stdout for "Count of GPU blocks" and "Block size". + Returns: (ready_bool, gpu_blocks, block_size, max_len_clamped, failure_reason) + """ + start = time.time() + gpu_blocks = 0 + block_size = 16 # default + max_len_clamped = None + + logs = [] + failure_reason = None + + while time.time() - start < timeout: + if process.poll() is not None: + # Process died. + for line in process.stdout: + line_str = line.decode("utf-8", errors="replace").strip() + logs.append(line_str) + + # SCAN FULL HISTORY if not found yet + # Sometimes error was in previous lines or split + if not failure_reason: + full_log = "\n".join(logs) + + # Check 1: Sampler OOM + if "warming up sampler" in full_log and "CUDA out of memory" in full_log: + failure_reason = "Sampler Warmup OOM" + + # Check 2: Explicit vLLM suggestion (Estimated) + # "estimated maximum model length is 127120" + elif "estimated maximum model length is" in full_log: + m = re.search(r"estimated maximum model length is (\d+)", full_log) + if m: + failure_reason = f"estimated maximum model length is {m.group(1)}" + + # Check 3: Derived Max Model Len + # "derived max_model_len (max_position_embeddings=131072.0 ...)" + elif "derived max_model_len" in full_log: + failure_reason = "derived max_model_len detected" + + # Check 4: Capacity/Value Error + elif "ValueError" in full_log and "maximum number of tokens" in full_log: + failure_reason = "Capacity Error (Found in history)" + + # Check 5: Generic OOM + elif "CUDA out of memory" in full_log or "hipErrorOutOfMemory" in full_log: + failure_reason = "OOM detected" + + if not failure_reason: + # Unexpected death! Dump logs to see why. + log("CRITICAL: Process died unexpectedly! Dumping last 100 lines:") + print("=== vLLM SERVER LOGS (LAST 100 LINES) ===") + for l in logs[-100:]: + print(l) + print("=============================================") + + return False, 0, 0, None, failure_reason + + line = process.stdout.readline() + if line: + line_str = line.decode("utf-8", errors="replace").strip() + logs.append(line_str) + + # 1. Parse Legacy "GPU blocks" (if present) + m_blocks = re.search(r"# GPU blocks:\s*(\d+)", line_str) + if m_blocks: + gpu_blocks = int(m_blocks.group(1)) + block_size = 16 # assume default unless found + log(f" -> Found GPU blocks: {gpu_blocks} (Legacy)") + + # 2. Parse Newer "GPU KV cache size" (vLLM 0.11+) + # "GPU KV cache size: 111,536 tokens" + m_kv_tokens = re.search(r"GPU KV cache size:\s*([\d,]+)\s*tokens", line_str) + if m_kv_tokens: + tokens_str = m_kv_tokens.group(1).replace(",", "") + gpu_blocks = int(tokens_str) # We use 'gpu_blocks' variable to store total tokens now for simplicity + block_size = 1 # Effectively 1 because we have the total count + log(f" -> Found GPU KV Cache tokens: {gpu_blocks}") + + # 3. Parse Block Size (optional, mostly for legacy) + m_bs = re.search(r"block_size=(\d+)", line_str) + if m_bs: + block_size = int(m_bs.group(1)) + + # Failure hints + if "ValueError" in line_str and "maximum number of tokens" in line_str: + failure_reason = line_str + if "derived max_model_len" in line_str: + failure_reason = line_str + if "warming up sampler" in line_str and "CUDA out of memory" in line_str: + failure_reason = "Sampler Warmup OOM" + elif "CUDA out of memory" in line_str or "hipErrorOutOfMemory" in line_str: + failure_reason = "OOM detected" + + # Check for startup + if "Application startup complete" in line_str or "Uvicorn running on" in line_str: + if gpu_blocks > 0: + log(" -> Server signal detected. Waiting 5s for socket stability...") + time.sleep(5) + return True, gpu_blocks, block_size, max_len_clamped, None + else: + return False, 0, 0, None, "Parsed Success but Token/Block Count was 0" + + # Timeout case + log("CRITICAL: Server startup timed out! Dumping last 100 lines:") + print("=== vLLM SERVER LOGS (LAST 100 LINES) ===") + for l in logs[-100:]: + print(l) + print("=============================================") + return False, 0, 0, None, "Timeout" + +def verify_context(model, context_len): + """ + Sends a request to the server with length ~context_len to verify stability. + """ + url = f"http://{HOST}:{PORT}/v1/completions" + + # We use a simple "A " * N prompt. + # Llama 3 tokenizer: "A" is usually 1 token. + + prompt = "A " * int(context_len * 0.5) # 50% fill to be safe/approx + + payload = { + "model": model, + "prompt": prompt, + "max_tokens": 10, + "temperature": 0 + } + + # Retry loop for connection refusals (race condition) + max_retries = 5 + for attempt in range(max_retries): + try: + # Increased timeout to 300s because prefilling 60k+ tokens takes time! + r = requests.post(url, json=payload, timeout=300) + if r.status_code == 200: + return True, "Success" + else: + # If 500 or 400 error, maybe we shouldn't retry? Usually yes for 500 if transient. + # But for now let's just fail or retry. + # If we are OOMing, we will likely get a 500 or it will hang. + return False, f"HTTP {r.status_code}: {r.text[:200]}" + except requests.exceptions.ConnectionError: + if attempt < max_retries - 1: + log(f" -> Connection refused. Retrying verification ({attempt+1}/{max_retries})...") + time.sleep(2) + else: + return False, "Connection Refused (Max Retries)" + except Exception as e: + return False, str(e) + + return False, "Unknown Error" + +def run_probe(model, tp, util, max_seqs, start_limit=None): + """ + Probes a specific configuration starting from the model's architectural limit. + """ + trust_remote = MODEL_TABLE[model].get("trust_remote", False) + # 1. Get the Advertised Limit (The "Smart" Way) + arch_limit = get_hf_context_limit(model, trust_remote) + + # Intelligent Start: If we know a lower limit worked for lower concurrency, start there. + target_len = arch_limit + if start_limit: + target_len = min(arch_limit, start_limit) + log(f" -> Smart Start: Capping initial probe at {target_len} (based on previous run)") + + result_data = { + "model": model, + "tp": tp, + "util": util, + "max_seqs": max_seqs, + "model_limit": arch_limit, + "configured_len": 0, + "real_capacity": 0, + "status": "fail", + "error": "" + } + + log(f"Probing {model} | TP={tp} | Util={util} | Seqs={max_seqs} | Model Limit={arch_limit}") + + # We loop until we succeed OR we drop below a useful context size. + while target_len >= 2048: + force_cleanup() + + cmd, env = get_vllm_server_cmd(model, tp, util, target_len, max_seqs) + log(f"DEBUG: Cmd: {' '.join(cmd)}") + + proc = None + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) + ready, blocks, block_size, _, fail_msg = wait_for_server_and_parse(proc) + + if ready: + # Success - but let's VERIFY it actually answers + total_capacity = blocks * block_size + workable_len = min(target_len, total_capacity) + + # Verify with actual request + log(f" -> Server ready. Verifying stability with approx {int(workable_len * 0.5)} tokens...") + v_ok, v_msg = verify_context(model, workable_len) + + if v_ok: + log(f" -> Success! capacity={total_capacity}, configured={workable_len}") + log(f" -> Verification passed: {v_msg}") + + # Cleanup SUCCESSFUL process immediately + proc.terminate() + try: proc.wait(timeout=5) + except: proc.kill() + + result_data["status"] = "success" + result_data["configured_len"] = target_len + result_data["real_capacity"] = total_capacity + result_data["max_context_1_user"] = workable_len + + return result_data + else: + log(f" -> Server started, but Verification FAILED: {v_msg}") + # Treat as a crash/failure, back off + fail_msg = "Verification Failed" + + # Capture any remaining logs if the process is dead or dying + # Or just read what's currently available non-blocking? + # Simpler: just terminate and read output. + proc.terminate() + try: + outs, errs = proc.communicate(timeout=5) + if outs: + print("=== vLLM SERVER LOGS (DURING VERIFICATION FAILURE) ===") + print(outs.decode('utf-8', errors='replace')) + print("======================================================") + except: + proc.kill() + + + # If we fall through here, ready=False OR verify=False + log(f" -> Attempt failed at {target_len}") + if fail_msg: log(f" Reason: {fail_msg}") + result_data["error"] = fail_msg if fail_msg else "Process died or timed out" + + if fail_msg: + # Case V: Verification Failed (Server up, but unstable inference) + # User requests drop to 0.95 tier immediately. + # Must check this FIRST to ensure we don't fall through. + if "Verification Failed" in str(fail_msg): + log(" -> Verification Failed (Unstable). Aborting this Util, dropping to lower tier.") + break + + # Case S: Sampler Warmup OOM (Fatal for this Util) + if "Sampler Warmup OOM" in fail_msg: + log(" -> Critical Sampler OOM. Utilization/Seqs too high. Aborting this configuration.") + break # Give up on this Util/Seq combo immediately + + # Case X: Dirty State / Zombie VRAM + # "Free memory on device (1.56/31.86 GiB) on startup is less than desired..." + if "Free memory on device" in fail_msg and "less than desired" in fail_msg: + log(" -> Dirty VRAM detected (previous run didn't cleanup?). Retrying with HARD cleanup.") + force_cleanup(hard=True) + continue # Retry SAME target_len + + # Case A: VRAM Limit ("maximum number of tokens... is X") + m_capacity = re.search(r"maximum number of tokens.*?KV cache is (\d+)", fail_msg) + if m_capacity: + cap = int(m_capacity.group(1)) + log(f" -> Found Hardware Capacity: {cap}") + target_len = cap + continue # Retry Exact Cap + + # Case B: Model Limit mismatch + # "Value error, User-specified max_model_len (500000) is greater than the derived max_model_len (max_position_embeddings=131072.0 ...)" + # We regex for 'derived max_model_len' and then look for numbers in the proximity. + + if "derived max_model_len" in fail_msg: + # Try to capture "max_position_embeddings=131072" + m_pos = re.search(r"max_position_embeddings=([\d\.]+)", fail_msg) + if m_pos: + limit = int(float(m_pos.group(1))) # handle 131072.0 + log(f" -> Found Model Limit: {limit}") + target_len = limit + continue + + # Fallback: look for simple parenthesis pattern if the above fails + m_derived = re.search(r"derived max_model_len\s*\((\d+)\)", fail_msg) + if m_derived: + limit = int(m_derived.group(1)) + log(f" -> Found Model Limit (Legacy): {limit}") + target_len = limit + continue + + # Case C: Estimated Max Length (New vLLM Safe Limit) + # "estimated maximum model length is 111536" + m_est = re.search(r"estimated maximum model length is (\d+)", fail_msg) + if m_est: + limit = int(m_est.group(1)) + log(f" -> Found vLLM Estimated Limit: {limit}") + target_len = limit + continue + + # Case D: Generic OOM/Crash + target_len = int(target_len * 0.8) + log(f" -> Backing off to: {target_len}") + + if target_len < 2048: + log(" -> Give up (too small)") + break + finally: + if proc: + try: proc.terminate() + except: pass + try: proc.kill() + except: pass + proc.wait() + force_cleanup() + + return result_data + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, help="Filter to run only this model (substring match)") + parser.add_argument("--steps", type=int, default=-1, help="Number of models to run (default: all)") + args = parser.parse_args() + + gpu_count = get_gpu_count() + + # 1. Load existing results to support RESUME + results = [] + if RESULTS_FILE.exists(): + try: + with open(RESULTS_FILE, "r") as f: + results = json.load(f) + log(f"Loaded {len(results)} previous results. Resuming...") + except Exception as e: + log(f"Warning: Could not read existing results: {e}") + + count = 0 + for model in MODELS_TO_RUN: + if args.model and args.model not in model: + continue + + config = MODEL_TABLE[model] + valid_tps = [t for t in config["valid_tp"] if t <= gpu_count] + + for tp in valid_tps: + # Track successful seqs for this TP to skip lower utils + # effectively: {seqs_count: max_working_util} + # Since we iterate high-util -> low-util, if we succeeded already for this 'seqs', we skip. + successful_seqs = set() + + # Reset smart limit for each TP (TP2 should not inherit TP1's limit) + last_working_len = None + + for util in GPU_UTIL_STEPS: + + for seqs in CONCURRENCY_STEPS: + if seqs in successful_seqs: + log(f"Skipping {model} (TP={tp}, Util={util}, Seqs={seqs}) - Already succeeded at higher util.") + continue + + # Check if we already have this result + existing_res = next((r for r in results + if r["model"] == model + and r["tp"] == tp + and str(r["util"]) == str(util) + and r["max_seqs"] == seqs), None) + + if existing_res: + res = existing_res + log(f"Skipping {model} (TP={tp}, Util={util}, Seqs={seqs}) - Found in results.") + else: + # New run + res = run_probe(model, tp, util, seqs, start_limit=last_working_len) + results.append(res) + + # Save immediately + with open(RESULTS_FILE, "w") as f: + json.dump(results, f, indent=2) + + # Update logic for Resume OR New Run: + if res["status"] == "success": + last_working_len = res["configured_len"] + successful_seqs.add(seqs) # Mark this seq count as done for this TP + + # Smart Break: If we failed at this concurrency level (capacity=0), + # higher concurrency will also fail. + if res["real_capacity"] == 0 or res["status"] == "fail": + log(f"Stopping higher concurrency tests for {model} (failed at {seqs} seqs)") + break + + count += 1 + if args.steps != -1 and count >= args.steps and not args.model: + break + + # generate_report(results) - Moved to separate script + +if __name__ == "__main__": + main() diff --git a/benchmarks/generate_readme_table.py b/benchmarks/generate_readme_table.py new file mode 100644 index 0000000..a3801cf --- /dev/null +++ b/benchmarks/generate_readme_table.py @@ -0,0 +1,119 @@ + +import json +from pathlib import Path + +RESULTS_FILE = Path("max_context_results.json") + +def format_context(val): + if val is None or val == 0: + return "Fail" + if val >= 1000: + return f"{val/1000:.0f}k" + return str(val) + +def main(): + if not RESULTS_FILE.exists(): + print(f"Error: {RESULTS_FILE} not found.") + return + + with open(RESULTS_FILE, "r") as f: + data = json.load(f) + + # Organize data: model -> tp -> seq -> list of (util, context) + tree = {} + for row in data: + if row.get("status") != "success": + continue + + model = row["model"] + tp = row["tp"] + util = float(row["util"]) + seqs = row["max_seqs"] + context = row.get("max_context_1_user", 0) + + if model not in tree: tree[model] = {} + if tp not in tree[model]: tree[model][tp] = {} + if seqs not in tree[model][tp]: tree[model][tp][seqs] = [] + + tree[model][tp][seqs].append((context, util)) + + # Define headers + # Moving Memory Utilization into the cells to allow per-concurrency variation + print("\n**Table Key:** Cell values represent `Max Context Length (GPU Memory Utilization)`.\n") + print("| Model | TP | 1 Req | 4 Reqs | 8 Reqs | 16 Reqs |") + print("| :--- | :--- | :--- | :--- | :--- | :--- |") + + model_order = [ + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "openai/gpt-oss-20b", + "RedHatAI/Qwen3-14B-FP8-dynamic", + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "RedHatAI/gemma-3-27b-it-FP8-dynamic", + ] + + for model_name in model_order: + if model_name not in tree: + continue + + tps = sorted(tree[model_name].keys()) + + for i, tp in enumerate(tps): + # 1. Gather best raw results for each concurrency level + best_by_seq = {} + seq_levels = [1, 4, 8, 16] + + for seq in seq_levels: + candidates = tree[model_name][tp].get(seq, []) + if not candidates: + continue + + # Sort criteria: Maximize Context, then Minimize Util + best = sorted(candidates, key=lambda x: (-x[0], x[1]))[0] + best_by_seq[seq] = best + + # 2. Smooth/Backfill: Ensure Ctx(reqs=low) >= Ctx(reqs=high) + # If 4 users can do 156k, 1 user certainly can too. + # We take the standard of "Better" = (Higher Context, Lower Util) + final_values = {} + for i_seq, seq in enumerate(seq_levels): + # Gather all valid results from this level and higher + valid_futures = [] + for future_seq in seq_levels[i_seq:]: + if future_seq in best_by_seq: + valid_futures.append(best_by_seq[future_seq]) + + if not valid_futures: + final_values[seq] = None + else: + # Pick the best among them + # best tuple -> max context, then min util + # key function for max(): (context, -util) + final_values[seq] = max(valid_futures, key=lambda x: (x[0], -x[1])) + + # 3. Format Output + row_cells = [] + + # Model Name + if i == 0: + row_cells.append(f"**`{model_name}`**") + else: + row_cells.append("") + + # TP + row_cells.append(str(tp)) + + # Concurrency Columns + for seq in seq_levels: + val = final_values.get(seq) + if val is None: + row_cells.append("Fail") + else: + ctx_val, util_val = val + row_cells.append(f"{format_context(ctx_val)} ({util_val:.2f})") + + print("| " + " | ".join(row_cells) + " |") + +if __name__ == "__main__": + main() diff --git a/benchmarks/max_context_results.json b/benchmarks/max_context_results.json new file mode 100644 index 0000000..db9d61c --- /dev/null +++ b/benchmarks/max_context_results.json @@ -0,0 +1,611 @@ +[ + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 1, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 127120, + "real_capacity": 127120, + "status": "success", + "error": "estimated maximum model length is 127120", + "max_context_1_user": 127120 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 1, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 127120, + "real_capacity": 127168, + "status": "success", + "error": "", + "max_context_1_user": 127120 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 1, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 127120, + "real_capacity": 127184, + "status": "success", + "error": "", + "max_context_1_user": 127120 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 1, + "util": "0.98", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 127120, + "real_capacity": 127152, + "status": "success", + "error": "", + "max_context_1_user": 127120 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 104857, + "real_capacity": 379984, + "status": "success", + "error": "Verification Failed", + "max_context_1_user": 104857 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 2, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 104857, + "real_capacity": 380080, + "status": "success", + "error": "", + "max_context_1_user": 104857 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 2, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 104857, + "real_capacity": 380112, + "status": "success", + "error": "", + "max_context_1_user": 104857 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "tp": 2, + "util": "0.98", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 104857, + "real_capacity": 380112, + "status": "success", + "error": "", + "max_context_1_user": 104857 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 1, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 355520, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 1, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 355312, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 1, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 355312, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 1, + "util": "0.98", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 354432, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 0, + "real_capacity": 0, + "status": "fail", + "error": "Verification Failed" + }, + { + "model": "openai/gpt-oss-20b", + "tp": 2, + "util": "0.95", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 973840, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 2, + "util": "0.95", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 973408, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 2, + "util": "0.95", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 973408, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "openai/gpt-oss-20b", + "tp": 2, + "util": "0.95", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 971776, + "status": "success", + "error": "", + "max_context_1_user": 131072 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 1, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 99024, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 4, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 98960, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 8, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 98928, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 16, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 98928, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 40960, + "configured_len": 0, + "real_capacity": 0, + "status": "fail", + "error": "Verification Failed" + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 1, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 287168, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 4, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 287040, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 8, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 287040, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 16, + "model_limit": 40960, + "configured_len": 40960, + "real_capacity": 286688, + "status": "success", + "error": "", + "max_context_1_user": 40960 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 1, + "util": "0.98", + "max_seqs": 1, + "model_limit": 262144, + "configured_len": 150560, + "real_capacity": 157312, + "status": "success", + "error": "estimated maximum model length is 150560", + "max_context_1_user": 150560 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 1, + "util": "0.98", + "max_seqs": 4, + "model_limit": 262144, + "configured_len": 150560, + "real_capacity": 157184, + "status": "success", + "error": "", + "max_context_1_user": 150560 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 1, + "util": "0.98", + "max_seqs": 8, + "model_limit": 262144, + "configured_len": 150560, + "real_capacity": 157136, + "status": "success", + "error": "", + "max_context_1_user": 150560 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 1, + "util": "0.98", + "max_seqs": 16, + "model_limit": 262144, + "configured_len": 150560, + "real_capacity": 157136, + "status": "success", + "error": "", + "max_context_1_user": 150560 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 485264, + "status": "success", + "error": "", + "max_context_1_user": 262144 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 4, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 492752, + "status": "success", + "error": "", + "max_context_1_user": 262144 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 8, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 492752, + "status": "success", + "error": "", + "max_context_1_user": 262144 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 16, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 492752, + "status": "success", + "error": "", + "max_context_1_user": 262144 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 145248, + "status": "success", + "error": "", + "max_context_1_user": 145248 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 4, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 156128, + "status": "success", + "error": "", + "max_context_1_user": 156128 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 8, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 156128, + "status": "success", + "error": "", + "max_context_1_user": 156128 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "tp": 2, + "util": "0.98", + "max_seqs": 16, + "model_limit": 262144, + "configured_len": 262144, + "real_capacity": 156128, + "status": "success", + "error": "", + "max_context_1_user": 156128 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 59728, + "status": "success", + "error": "", + "max_context_1_user": 59728 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 59760, + "status": "success", + "error": "", + "max_context_1_user": 59760 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 59760, + "status": "success", + "error": "", + "max_context_1_user": 59760 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 59760, + "status": "success", + "error": "", + "max_context_1_user": 59760 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 44368, + "status": "success", + "error": "", + "max_context_1_user": 44368 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 44784, + "status": "success", + "error": "", + "max_context_1_user": 44784 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 44784, + "status": "success", + "error": "", + "max_context_1_user": 44784 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 1, + "util": "0.98", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 44784, + "status": "success", + "error": "", + "max_context_1_user": 44784 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 1, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 126240, + "status": "success", + "error": "", + "max_context_1_user": 126240 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 4, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 126288, + "status": "success", + "error": "", + "max_context_1_user": 126288 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 2, + "util": "0.98", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 0, + "real_capacity": 0, + "status": "fail", + "error": "Verification Failed" + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 8, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 121072, + "status": "success", + "error": "", + "max_context_1_user": 121072 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "tp": 2, + "util": "0.95", + "max_seqs": 16, + "model_limit": 131072, + "configured_len": 131072, + "real_capacity": 121072, + "status": "success", + "error": "", + "max_context_1_user": 121072 + } +] \ No newline at end of file diff --git a/benchmarks/run_vllm_bench.py b/benchmarks/run_vllm_bench.py new file mode 100644 index 0000000..4676373 --- /dev/null +++ b/benchmarks/run_vllm_bench.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +import subprocess, time, json, sys, os, requests, re, argparse +from pathlib import Path + +# ========================= +# ⚙️ GLOBAL SETTINGS +# ========================= + +# HARDWARE: 2x AMD Radeon AI PRO R9700 (32GB, RDNA 4) +GPU_UTIL = "0.98" +PORT = 8000 +HOST = "127.0.0.1" + +# BENCHMARK TOGGLES +# AITER is disabled/removed. + + +# 1. THROUGHPUT CONFIG +OFF_NUM_PROMPTS = 1000 +OFF_FORCED_OUTPUT = "512" +# Default fallback if not specified in MODEL_TABLE +DEFAULT_BATCH_TOKENS = "8192" + +# 2. LATENCY CONFIG +SRV_DURATION = 180 +QPS_SWEEP = [1.0, 4.0] + +# Fallbacks +FALLBACK_INPUT_LEN = 1024 +FALLBACK_OUTPUT_LEN = 512 + +RESULTS_DIR = Path("benchmark_results") +RESULTS_DIR.mkdir(exist_ok=True) + +# ========================= +# 🛠️ MODEL CONFIGURATION 🛠️ +# ========================= + +MODEL_TABLE = { + # 1. Llama 3.1 8B Instruct + # MAD uses 131k tokens. We scale to 32k for 32GB VRAM safety. + "meta-llama/Meta-Llama-3.1-8B-Instruct": { + "ctx": "65536", + "trust_remote": False, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + + # 2. GPT-OSS 20B (MXFP4) + # MAD Row 0 uses 8192. We match this exactly. + "openai/gpt-oss-20b": { + "ctx": "32768", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "8192" + }, + + # 3. Qwen 14B FP8 + # MAD uses 40k. We use 32k. + "RedHatAI/Qwen3-14B-FP8-dynamic": { + "ctx": "32768", + "trust_remote": True, + "valid_tp": [1], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + + # 4. Qwen 30B 4-bit + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": { + "ctx": "24576", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + + # 5. Qwen 80B AWQ (The Big One) [NEW] + # Size: ~48GB. Fits on 2x32GB (64GB). Leftover for Cache: ~16GB. + # Config: 20k ctx fits in that cache. Eager mode required for stability. + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { + "ctx": "20480", + "trust_remote": True, + "valid_tp": [2], # Too big for single GPU + "max_num_seqs": "32", # Lower concurrency for safety + "max_tokens": "16384", # Lower batch size because Eager mode is CPU intensive + "enforce_eager": False, + "env": {"VLLM_USE_TRITON_AWQ": "1"} # Fixes "Unsupported Hardware" error + }, + + # 76 Gemma 3 27B FP8 + "RedHatAI/gemma-3-27b-it-FP8-dynamic": { + "ctx": "29000", + "trust_remote": True, + "valid_tp": [2], + "max_num_seqs": "32", + "max_tokens": "29000", + "gpu_util": "0.94", + }, + + # 7. Gemma 3 12B FP8 + "RedHatAI/gemma-3-12b-it-FP8-dynamic": { + "ctx": "9900", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "9900", + }, +} + +MODELS_TO_RUN = [ + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "openai/gpt-oss-20b", + "RedHatAI/Qwen3-14B-FP8-dynamic", + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "RedHatAI/gemma-3-12b-it-FP8-dynamic", +] + +# ========================= +# UTILS +# ========================= + +def log(msg): print(f"\n[BENCH] {msg}") + +def get_gpu_count(): + try: + # Using rocm-smi --showid to list GPUs. + # Output format: "GPU[0] : Device Name: ..." + res = subprocess.run(["rocm-smi", "--showid"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode == 0: + # Filter specifically for the target GPU as requested + target_gpu = "AMD Radeon AI PRO R9700" + count = 0 + for line in res.stdout.strip().split('\n'): + if "Device Name" in line and target_gpu in line: + count += 1 + + return count if count > 0 else 1 + else: + log("rocm-smi failed, defaulting to 2 GPUs (Hardcoded Fallback)") + return 2 + except Exception as e: + log(f"Error detecting GPUs: {e}, defaulting to 2 GPUs") + return 2 + +def kill_vllm(): + subprocess.run("pgrep -f 'vllm serve' | xargs -r kill -9", + shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(5) + +def nuke_vllm_cache(): + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + time.sleep(2) + except: pass + +def get_dataset(): + data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json") + if data_path.exists(): return str(data_path) + + log("Downloading ShareGPT dataset...") + url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + try: + r = requests.get(url, stream=True, timeout=15) + r.raise_for_status() + with open(data_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): f.write(chunk) + return str(data_path) + except Exception as e: + log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.") + return None + +def wait_for_server(url, process, timeout=600): + start = time.time() + while time.time() - start < timeout: + if process.poll() is not None: + log(f"CRITICAL: Server died! Ret: {process.returncode}") + return False + try: + if requests.get(f"{url}/v1/models", timeout=2).status_code == 200: + log("Server ready. Stabilizing...") + time.sleep(5) + return True + except: pass + time.sleep(2) + return False + +def get_model_args(model, tp_size): + config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"}) + + # Allow per-model GPU utilization override + util = config.get("gpu_util", GPU_UTIL) + + cmd = [ + "--model", model, + "--gpu-memory-utilization", util, + "--max-model-len", config["ctx"], + "--dtype", "auto", + "--tensor-parallel-size", str(tp_size), + "--max-num-seqs", config["max_num_seqs"] + ] + + if config.get("trust_remote"): cmd.append("--trust-remote-code") + if config.get("enforce_eager"): cmd.append("--enforce-eager") + + return cmd + +def run_throughput(model, tp_size): + if tp_size not in MODEL_TABLE[model]["valid_tp"]: return + + model_safe = model.replace("/", "_") + output_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_throughput.json" + + if output_file.exists(): + log(f"SKIP Throughput {model} (TP={tp_size})") + return + + dataset_path = get_dataset() + dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"] + + # Retrieve Model-Specific Batch Tokens + batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS) + + log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...") + kill_vllm() + nuke_vllm_cache() + + cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size) + cmd.extend([ + "--num-prompts", str(OFF_NUM_PROMPTS), + "--max-num-batched-tokens", batch_tokens, + "--output-len", OFF_FORCED_OUTPUT, + "--output-json", str(output_file), + "--disable-log-stats" + ]) + cmd.extend(dataset_args) + + # ENV Setup: Global + Model Specific + env = os.environ.copy() + + # Inject model specific env vars (e.g. for AWQ) + model_env = MODEL_TABLE[model].get("env", {}) + env.update(model_env) + + try: + subprocess.run(cmd, check=True, env=env) + except: + log(f"ERROR: Throughput failed {model}") + +def run_latency(model, tp_size): + if tp_size not in MODEL_TABLE[model]["valid_tp"]: return + model_safe = model.replace("/", "_") + + if all((RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{q}_latency.json").exists() for q in QPS_SWEEP): + return + + dataset_path = get_dataset() + log(f"START Server {model} (TP={tp_size})...") + kill_vllm() + nuke_vllm_cache() + + srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w") + srv_args = [x for x in get_model_args(model, tp_size) if x != "--model" and x != model] + + # ENV Setup: Global + Model Specific + env = os.environ.copy() + + model_env = MODEL_TABLE[model].get("env", {}) + env.update(model_env) + + proc = subprocess.Popen(["vllm", "serve", model] + srv_args + ["--host", HOST, "--port", str(PORT)], + stdout=srv_log, stderr=srv_log, env=env) + + try: + if not wait_for_server(f"http://{HOST}:{PORT}", proc): return + + for qps in QPS_SWEEP: + out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json" + if out_file.exists(): continue + + log(f"BENCH QPS={qps}...") + bench_cmd = [ + "vllm", "bench", "serve", + "--model", model, + "--base-url", f"http://{HOST}:{PORT}", + "--request-rate", str(qps), + "--num-prompts", str(int(max(10, SRV_DURATION * qps))), + "--trust-remote-code" + ] + + if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path]) + else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"]) + + res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env) + with open(out_file, "w") as f: + f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2)) + + except Exception as e: log(f"CRASH: {e}") + finally: + proc.terminate() + kill_vllm() + +def print_summary(tps): + print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}") + print("-" * 105) + + for m in MODELS_TO_RUN: + msafe = m.replace("/", "_") + for tp in tps: + if tp not in MODEL_TABLE[m]["valid_tp"]: continue + + try: + tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text()) + tok_s = f"{tdata.get('tokens_per_second', 0):.1f}" + except: tok_s = "N/A" + + first_row = True + for q in QPS_SWEEP: + try: + ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text()) + raw = ldata["raw_output"] + ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1) + tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1) + except: ttft, tpot = "-", "-" + + name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else "" + + print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}") + first_row = False + print("-" * 105) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--tp", type=int, nargs="+", default=[1, 2]) + args = parser.parse_args() + + gpu_count = get_gpu_count() + log(f"Detected {gpu_count} AMD GPU(s)") + + valid_tp_args = [t for t in args.tp if t <= gpu_count] + if not valid_tp_args: + log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.") + sys.exit(0) + + kill_vllm() + for tp in valid_tp_args: + for m in MODELS_TO_RUN: + run_throughput(m, tp) + run_latency(m, tp) + print_summary(valid_tp_args) \ No newline at end of file diff --git a/benchmarks/run_vllm_bench_nvidia.py b/benchmarks/run_vllm_bench_nvidia.py new file mode 100644 index 0000000..9282b8f --- /dev/null +++ b/benchmarks/run_vllm_bench_nvidia.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +import subprocess, time, json, sys, os, requests, re, argparse +from pathlib import Path + +# ========================= +# ⚙️ GLOBAL SETTINGS +# ========================= + +# HARDWARE: NVIDIA GPUs (Auto-detected) +GPU_UTIL = "0.95" +PORT = 8000 +HOST = "127.0.0.1" + +# 1. THROUGHPUT CONFIG +OFF_NUM_PROMPTS = 1000 +OFF_FORCED_OUTPUT = "512" +# Default fallback if not specified in MODEL_TABLE +DEFAULT_BATCH_TOKENS = "8192" + +# 2. LATENCY CONFIG +SRV_DURATION = 180 +QPS_SWEEP = [1.0, 4.0] + +# Fallbacks +FALLBACK_INPUT_LEN = 1024 +FALLBACK_OUTPUT_LEN = 512 + +RESULTS_DIR = Path("benchmark_results_nvidia") +RESULTS_DIR.mkdir(exist_ok=True) + +# ========================= +# 🛠️ MODEL CONFIGURATION 🛠️ +# ========================= + +MODEL_TABLE = { + # 1. Llama 3.1 8B Instruct + "meta-llama/Meta-Llama-3.1-8B-Instruct": { + "ctx": "65536", + "trust_remote": False, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + + # 2. GPT-OSS 20B (MXFP4) + "openai/gpt-oss-20b": { + "ctx": "32768", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "8192", + }, + + # 3. Qwen 14B FP8 + "RedHatAI/Qwen3-14B-FP8-dynamic": { + "ctx": "32768", + "trust_remote": True, + "valid_tp": [1], + "max_num_seqs": "64", + "max_tokens": "32768", + "gpu_util": "0.90" + }, + + # 4. Qwen 30B 4-bit + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": { + "ctx": "24576", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + "gpu_util": "0.90" + }, + + # 5. Qwen 80B AWQ + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { + "ctx": "20480", + "trust_remote": True, + "valid_tp": [2], # Requires 2 GPUs + "max_num_seqs": "32", + "max_tokens": "16384", + }, + + # 6. Llama 3.1 8B FP8 + "RedHatAI/Llama-3.1-8B-Instruct-FP8-block": { + "ctx": "65536", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + }, + + # 7. Gemma 3 12B FP8 + "RedHatAI/gemma-3-12b-it-FP8-dynamic": { + "ctx": "32768", + "trust_remote": True, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768", + }, +} + +MODELS_TO_RUN = [ + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "openai/gpt-oss-20b", + "RedHatAI/Qwen3-14B-FP8-dynamic", + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "RedHatAI/gemma-3-12b-it-FP8-dynamic", +] + +# ========================= +# UTILS +# ========================= + +def log(msg): print(f"\n[BENCH] {msg}") + +def get_gpu_count(): + try: + # Using nvidia-smi -L to list GPUs + res = subprocess.run(["nvidia-smi", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode == 0: + count = len([line for line in res.stdout.strip().split('\n') if line.strip()]) + return count + else: + log("nvidia-smi failed, defaulting to 1 GPU") + return 1 + except Exception as e: + log(f"Error detecting GPUs: {e}, defaulting to 1 GPU") + return 1 + +def force_gpu_cleanup(): + """Simple cleanup: just kill vllm processes (excluding self).""" + try: + my_pid = os.getpid() + # Kill everything matching vllm EXCEPT this process + subprocess.run(f"pgrep -f 'vllm' | grep -v {my_pid} | xargs -r kill -9", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Original cleanups for other helpers + subprocess.run("pkill -9 -f 'multiprocessing'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run("pkill -9 -f 'resource_tracker'", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Try finding fuser to kill processes attached to device files + if subprocess.run("which fuser", shell=True, stdout=subprocess.DEVNULL).returncode == 0: + subprocess.run("fuser -k -9 /dev/nvidia*", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except: pass + time.sleep(5) + +def nuke_vllm_cache(): + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + time.sleep(2) + except: pass + +def get_dataset(): + data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json") + if data_path.exists(): return str(data_path) + + log("Downloading ShareGPT dataset...") + url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + try: + r = requests.get(url, stream=True, timeout=15) + r.raise_for_status() + with open(data_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): f.write(chunk) + return str(data_path) + except Exception as e: + log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.") + return None + +def wait_for_server(url, process, timeout=600): + start = time.time() + while time.time() - start < timeout: + if process.poll() is not None: + log(f"CRITICAL: Server died! Ret: {process.returncode}") + return False + try: + if requests.get(f"{url}/v1/models", timeout=2).status_code == 200: + log("Server ready. Stabilizing...") + time.sleep(5) + return True + except: pass + time.sleep(2) + return False + +# ========================= +# HARDWARE DETECTION (24GB vs 32GB) +# ========================= +def is_24gb_card(): + try: + res = subprocess.run(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True) + # Check first GPU memory + mem = int(res.stdout.strip().split('\n')[0]) + return mem < 28000 # 4090 is ~24576 + except: + return False + +IS_24GB = is_24gb_card() +if IS_24GB: log("Detected 24GB GPU class (e.g. RTX 4090). Applying memory overrides.") +else: log("Detected 32GB+ GPU class. Using standard config.") + +def get_model_args(model, tp_size): + config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"}) + + current_ctx = config["ctx"] + current_seqs = config["max_num_seqs"] + util = None + + if IS_24GB: + if model == "meta-llama/Meta-Llama-3.1-8B-Instruct": + current_ctx = "31800" + log(f"Override: Llama 8B ctx reduced to {current_ctx} for 24GB VRAM") + elif model == "openai/gpt-oss-20b": + current_ctx = "16384" + current_seqs = "32" + util = "0.90" + log(f"Override: GPT-20B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM") + elif model == "RedHatAI/Qwen3-14B-FP8-dynamic": + current_ctx = "4096" + current_seqs = "32" + util = "0.86" + log(f"Override: Qwen 14B ctx reduced to {current_ctx}, seqs to {current_seqs}, util to {util} for 24GB VRAM") + + if util is None: util = config.get("gpu_util", GPU_UTIL) + + cmd = [ + "--model", model, + "--gpu-memory-utilization", util, + "--max-model-len", current_ctx, + "--dtype", "auto", + "--tensor-parallel-size", str(tp_size), + "--max-num-seqs", current_seqs + ] + + if config.get("trust_remote"): cmd.append("--trust-remote-code") + if config.get("enforce_eager"): cmd.append("--enforce-eager") + + return cmd + +def run_throughput(model, tp_size): + if tp_size not in MODEL_TABLE[model]["valid_tp"]: return + + model_safe = model.replace("/", "_") + output_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_throughput.json" + + if output_file.exists(): + log(f"SKIP Throughput {model} (TP={tp_size})") + return + + dataset_path = get_dataset() + dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"] + + batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS) + + log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...") + force_gpu_cleanup() + nuke_vllm_cache() + + cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size) + cmd.extend([ + "--num-prompts", str(OFF_NUM_PROMPTS), + "--max-num-batched-tokens", batch_tokens, + "--output-len", OFF_FORCED_OUTPUT, + "--output-json", str(output_file), + "--disable-log-stats" + ]) + cmd.extend(dataset_args) + + env = os.environ.copy() + env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True" + model_env = MODEL_TABLE[model].get("env", {}) + env.update(model_env) + + ids_cmd = " ".join(cmd) + log(f"CMD: {ids_cmd}") + + try: + subprocess.run(cmd, check=True, env=env) + except: + log(f"ERROR: Throughput failed {model}") + +def run_latency(model, tp_size): + if tp_size not in MODEL_TABLE[model]["valid_tp"]: return + model_safe = model.replace("/", "_") + + if all((RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{q}_latency.json").exists() for q in QPS_SWEEP): + return + + dataset_path = get_dataset() + log(f"START Server {model} (TP={tp_size})...") + force_gpu_cleanup() + nuke_vllm_cache() + + + srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w") + + # Use get_model_args directly. It includes ["--model", model, ...] + srv_args = get_model_args(model, tp_size) + + env = os.environ.copy() + env["PYTORCH_ALLOC_CONF"] = "expandable_segments:True" + model_env = MODEL_TABLE[model].get("env", {}) + env.update(model_env) + + # Command: vllm serve --model ... (no positional model arg) + cmd = ["vllm", "serve"] + srv_args + ["--host", HOST, "--port", str(PORT)] + + ids_cmd = " ".join(cmd) + log(f"CMD: {ids_cmd}") + + proc = subprocess.Popen(cmd, stdout=srv_log, stderr=srv_log, env=env) + + try: + if not wait_for_server(f"http://{HOST}:{PORT}", proc): return + + for qps in QPS_SWEEP: + out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json" + if out_file.exists(): continue + + log(f"BENCH QPS={qps}...") + bench_cmd = [ + "vllm", "bench", "serve", + "--model", model, + "--base-url", f"http://{HOST}:{PORT}", + "--request-rate", str(qps), + "--num-prompts", str(int(max(10, SRV_DURATION * qps))), + "--trust-remote-code" + ] + + if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path]) + else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"]) + + res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env) + with open(out_file, "w") as f: + f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2)) + + except Exception as e: log(f"CRASH: {e}") + finally: + proc.terminate() + force_gpu_cleanup() + +def print_summary(tps): + print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}") + print("-" * 105) + + for m in MODELS_TO_RUN: + msafe = m.replace("/", "_") + for tp in tps: + if tp not in MODEL_TABLE[m]["valid_tp"]: continue + + try: + tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text()) + tok_s = f"{tdata.get('tokens_per_second', 0):.1f}" + except: tok_s = "N/A" + + first_row = True + for q in QPS_SWEEP: + try: + ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text()) + raw = ldata["raw_output"] + ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1) + tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1) + except: ttft, tpot = "-", "-" + + name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else "" + + print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}") + first_row = False + print("-" * 105) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--tp", type=int, nargs="+", default=[1, 2]) + args = parser.parse_args() + + gpu_count = get_gpu_count() + log(f"Detected {gpu_count} GPU(s)") + + valid_tp_args = [t for t in args.tp if t <= gpu_count] + if not valid_tp_args: + log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.") + sys.exit(0) + + force_gpu_cleanup() + for tp in valid_tp_args: + for m in MODELS_TO_RUN: + run_throughput(m, tp) + run_latency(m, tp) + print_summary(valid_tp_args) diff --git a/demo.gif b/demo.gif new file mode 100644 index 0000000..e303b31 Binary files /dev/null and b/demo.gif differ diff --git a/docs/assets/index2.css b/docs/assets/index2.css new file mode 100644 index 0000000..48d0e15 --- /dev/null +++ b/docs/assets/index2.css @@ -0,0 +1,401 @@ +:root { + --bg: #f8fafc; + --surface: #ffffff; + --text: #0f172a; + --text-sub: #64748b; + --accent: #d90007; + /* AMD Red */ + --accent-fade: #fff0f0; + --border: #e2e8f0; + --font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font-family: var(--font); + display: flex; + flex-direction: column; + height: 100vh; + overflow-y: auto; +} + +header, +.controls, +.panel-split, +#tables { + max-width: 860px; + margin: 0 auto; + width: 100%; + box-sizing: border-box; +} + +table { + border-collapse: collapse; + width: 100%; + table-layout: fixed; +} + + +h1 { + margin: 0; + font-size: 1.25rem; + font-weight: 600; +} + +p { + margin: 4px 0 0; + font-size: 0.875rem; + color: var(--text-sub); +} + +.legend { + margin-top: 12px; + display: flex; + align-items: center; + gap: 12px; +} + +.legend label { + font-size: 0.75rem; + font-weight: 600; + text-transform: uppercase; + color: var(--text-sub); +} + +.legend-pills { + display: flex; + gap: 8px; +} + +.legend-pill { + cursor: default !important; +} + +.legend-pill-default::before { + content: ""; + display: inline-block; + width: 8px; + height: 8px; + background: #cbd5e1; + border-radius: 50%; + margin-right: 6px; +} + +.legend-pill-dual::before { + content: ""; + display: inline-block; + width: 8px; + height: 8px; + background: #d90007; + border-radius: 50%; + margin-right: 6px; +} + +.controls { + background: var(--surface); + border-bottom: 1px solid var(--border); + padding: 12px 24px; + display: flex; + gap: 24px; + align-items: center; + flex-shrink: 0; +} + +.control { + display: flex; + flex-direction: column; + gap: 4px; +} + +.control label { + font-size: 0.75rem; + font-weight: 600; + color: var(--text-sub); +} + +input[type="text"], +select { + padding: 6px 10px; + border: 1px solid var(--border); + border-radius: 4px; + font-size: 0.875rem; + background: var(--bg); + min-width: 180px; +} + +.range-wrap { + position: relative; + width: 200px; + height: 20px; +} + +.range-track { + position: absolute; + top: 50%; + left: 0; + right: 0; + height: 4px; + background: #e3e7f1; + border-radius: 2px; + transform: translateY(-50%); +} + +input[type=range] { + position: absolute; + width: 100%; + pointer-events: none; + appearance: none; + background: none; + margin: 0; + top: 50%; + transform: translateY(-50%); +} + +input[type=range]::-webkit-slider-thumb { + pointer-events: auto; + appearance: none; + width: 16px; + height: 16px; + border-radius: 50%; + background: var(--surface); + border: 2px solid var(--accent); + cursor: pointer; +} + +.range-values { + font-size: 0.75rem; + color: var(--text-sub); + margin-top: 4px; + text-align: center; +} + +.panel { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.panel.compact { + flex: 0 0 auto; +} + +#tables-panel { + flex: 1; + background: var(--bg); + padding: 0; + overflow-y: auto; + margin-top: 24px; + margin-bottom: 40px; +} + +.panel-split { + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px 24px; + background: var(--surface); + border-bottom: 1px solid var(--border); +} + +.backend-header { + display: flex; + flex-direction: column; + gap: 8px; +} + +.backend-label { + display: flex; + align-items: center; + gap: 12px; +} + +.backend-label label { + font-size: 0.75rem; + font-weight: 600; + color: var(--text-sub); + text-transform: uppercase; +} + +.backend-list { + display: flex; + gap: 16px; + flex-wrap: wrap; +} + +.backend-item { + display: flex; + align-items: center; + gap: 6px; + font-size: 0.875rem; + cursor: pointer; + user-select: none; +} + +.test-block { + margin-bottom: 32px; + background: var(--surface); + border-top: 1px solid var(--border); + border-bottom: 1px solid var(--border); +} + +h2 { + padding: 16px 24px; + margin: 0; + font-size: 1rem; + background: #f1f5f9; + color: var(--text); + border-bottom: 1px solid var(--border); +} + +.table-wrap { + position: relative; + overflow: hidden; +} + +.table-scroll { + overflow-x: auto; + padding-bottom: 12px; + /* Scrollbar space */ +} + +/* ... */ + +.best { + background: #f0fdf4; +} + +.cell-error { + color: #ef4444; + font-size: 0.75rem; +} + +.cell-empty { + color: var(--border); + font-size: 0.75rem; + font-style: italic; +} + +/* Resize Overlay */ +.resize-overlay { + position: absolute; + top: 0; + left: 0; + pointer-events: none; + z-index: 5; +} + +.resize-bar { + position: absolute; + top: 0; + width: 6px; + height: 100%; + cursor: col-resize; + pointer-events: auto; + /* invisible usually, but can hover */ +} + +.resize-bar:hover { + background: rgba(0, 0, 0, 0.05); +} + +.resize-handle { + position: absolute; + right: 0; + top: 0; + bottom: 0; + width: 4px; + cursor: col-resize; +} + +.backend-header.dragging { + opacity: 0.5; +} + +.backend-header.drop-target { + border-left: 2px solid var(--accent); +} + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: #cbd5e1; + border-radius: 4px; +} + +/* Modal Styles */ +.modal { + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + background: rgba(0, 0, 0, 0.4); + display: flex; + justify-content: center; + align-items: center; + z-index: 1000; + opacity: 1; + transition: opacity 0.2s; +} + +.modal.hidden { + opacity: 0; + pointer-events: none; +} + +.modal-content { + background: var(--surface); + padding: 24px 32px; + border-radius: 8px; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15); + max-width: 500px; + width: 90%; + position: relative; + transform: translateY(0); + transition: transform 0.2s; +} + +.modal.hidden .modal-content { + transform: translateY(20px); +} + +.modal-close { + position: absolute; + top: 12px; + right: 12px; + background: transparent; + border: none; + font-size: 1.5rem; + line-height: 1; + color: var(--text-sub); + cursor: pointer; + padding: 4px; +} + +.modal-close:hover { + color: var(--text); +} + +.modal-content h2 { + margin-top: 0; + background: none; + border: none; + padding: 0; + font-size: 1.25rem; + margin-bottom: 12px; +} + +.modal-content p { + margin-bottom: 12px; + line-height: 1.5; +} \ No newline at end of file diff --git a/docs/assets/index2.js b/docs/assets/index2.js new file mode 100644 index 0000000..2825a67 --- /dev/null +++ b/docs/assets/index2.js @@ -0,0 +1,542 @@ +const K_SIGMA = 1.0; +const MIN_TOL = 0.25; +const MODEL_COL_WIDTH = 300; +// Winner column removed + +const state = { + envs: ["TP1", "TP2"], + backendOrder: ["TP1", "TP2"], + columnWidths: { "TP1": 260, "TP2": 260 }, + filters: { + search: "", + quant: "", + backends: new Set(["TP1", "TP2"]), + sizeLo: null, + sizeHi: null, + }, + ui: {}, + sizeStats: { min: Infinity, max: -Infinity }, + draggingEnv: null, + quantOptions: [], +}; + +document.addEventListener("DOMContentLoaded", async () => { + cacheUI(); + setupModals(); + try { + const res = await fetch("results.json"); + const data = await res.json(); + prepareData(data?.runs || []); + initializeControls(); + renderTables(); + } catch (err) { + console.error("Failed to load results.json", err); + state.ui.stats.textContent = "Failed to load results.json"; + } +}); + +function cacheUI() { + state.ui = { + search: document.getElementById("filter-search"), + quant: document.getElementById("filter-quant"), + backendList: document.getElementById("backend-list"), + backendAll: document.getElementById("backend-all"), + backendNone: document.getElementById("backend-none"), + sizeLo: document.getElementById("sizeLo"), + sizeHi: document.getElementById("sizeHi"), + sizeTrack: document.getElementById("sizeTrack"), + sizeLoVal: document.getElementById("sizeLoVal"), + sizeHiVal: document.getElementById("sizeHiVal"), + stats: document.getElementById("stats-line"), + resetBtn: document.getElementById("reset-layout"), + tables: document.getElementById("tables"), + // Modal hooks + tp1ModalOpen: document.getElementById("tp1-modal-open"), + tp2ModalOpen: document.getElementById("tp2-modal-open"), + tp1Modal: document.getElementById("tp1-modal"), + tp2Modal: document.getElementById("tp2-modal"), + tp1ModalClose: document.getElementById("tp1-modal-close"), + tp2ModalClose: document.getElementById("tp2-modal-close"), + }; +} + +function setupModals() { + const modalConfigs = [ + { open: state.ui.tp1ModalOpen, modal: state.ui.tp1Modal, close: state.ui.tp1ModalClose }, + { open: state.ui.tp2ModalOpen, modal: state.ui.tp2Modal, close: state.ui.tp2ModalClose }, + ]; + + modalConfigs.forEach(({ open, modal, close }) => { + if (!open || !modal) return; + const openModal = () => modal.classList.remove("hidden"); + const closeModal = () => modal.classList.add("hidden"); + open.addEventListener("click", openModal); + close?.addEventListener("click", closeModal); + modal.addEventListener("click", (e) => { + if (e.target === modal) closeModal(); + }); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && !modal.classList.contains("hidden")) { + closeModal(); + } + }); + }); +} + +function prepareData(runs) { + const quantSet = new Set(); + // Tests map: TestName -> { name: ..., models: Map(ModelName -> Row) } + const testsMap = new Map(); + + for (const run of runs) { + if (!run.test) continue; + const testKey = run.test; + + if (run.quant) quantSet.add(run.quant.toUpperCase()); + + if (!testsMap.has(testKey)) { + testsMap.set(testKey, { name: testKey, models: new Map() }); + } + const testEntry = testsMap.get(testKey); + + const modelName = run.model_clean || run.model; + + if (!testEntry.models.has(modelName)) { + testEntry.models.set(modelName, { + model: modelName, + quant: (run.quant || "Unknown").toUpperCase(), + sizeB: run.name_params_b ?? run.params_b ?? null, + backends: {}, + search_blob: [modelName, run.quant, run.env, run.test] + .filter(Boolean) + .map((s) => s.toString().toLowerCase()) + .join(" "), + }); + } + + const row = testEntry.models.get(modelName); + + // Update stats + if (row.sizeB != null) { + state.sizeStats.min = Math.min(state.sizeStats.min, row.sizeB); + state.sizeStats.max = Math.max(state.sizeStats.max, row.sizeB); + } + + // Add backend data + // run.env comes from python script as "TP1" or "TP2" + const env = run.env; + row.backends[env] = { + mean: typeof run.tps_mean === "number" ? run.tps_mean : null, + std: 0, // Not currently parsed + error: Boolean(run.error), + error_type: run.error_type || null, + }; + } + + state.tests = [...testsMap.values()].sort((a, b) => a.name.localeCompare(b.name)); + state.quantOptions = [...quantSet].sort(); +} + +function initializeControls() { + const { quant, backendList, search, resetBtn, sizeLo, sizeHi } = state.ui; + + quant.innerHTML = ""; + const anyOpt = document.createElement("option"); + anyOpt.value = ""; + anyOpt.textContent = "Any"; + quant.appendChild(anyOpt); + state.quantOptions.forEach((q) => { + const opt = document.createElement("option"); + opt.value = q; + opt.textContent = q; + quant.appendChild(opt); + }); + + renderBackendList(); + setupSizeSlider(); + + search.addEventListener("input", (e) => { + state.filters.search = (e.target.value || "").trim().toLowerCase(); + renderTables(); + }); + + quant.addEventListener("change", (e) => { + state.filters.quant = e.target.value; + renderTables(); + }); + + backendList.addEventListener("change", (e) => { + const checkbox = e.target.closest("input[data-env]"); + if (!checkbox) return; + const env = checkbox.dataset.env; + if (checkbox.checked) { + state.filters.backends.add(env); + } else { + state.filters.backends.delete(env); + } + renderTables(); + }); + + state.ui.backendAll.addEventListener("click", () => { + state.filters.backends = new Set(state.envs); + renderBackendList(); + renderTables(); + }); + + state.ui.backendNone.addEventListener("click", () => { + state.filters.backends = new Set(); + renderBackendList(); + renderTables(); + }); + + sizeLo.addEventListener("input", () => updateSizeUI(true)); + sizeHi.addEventListener("input", () => updateSizeUI(true)); + + resetBtn.addEventListener("click", () => { + state.filters.search = ""; + state.filters.quant = ""; + state.filters.backends = new Set(state.envs); + search.value = ""; + quant.value = ""; + renderBackendList(); + setupSizeSlider(); + renderTables(); + }); +} + +function renderBackendList() { + const container = state.ui.backendList; + container.innerHTML = ""; + state.backendOrder.forEach((env) => { + const label = document.createElement("label"); + label.className = "backend-item"; + const checkbox = document.createElement("input"); + checkbox.type = "checkbox"; + checkbox.dataset.env = env; + checkbox.checked = state.filters.backends.has(env); + label.appendChild(checkbox); + + const baseSpan = document.createElement("span"); + baseSpan.textContent = env; + label.appendChild(baseSpan); + + container.appendChild(label); + }); +} + +function setupSizeSlider() { + const { sizeLo, sizeHi } = state.ui; + const minRaw = state.sizeStats.min === Infinity ? 0 : Math.floor(state.sizeStats.min || 0); + const maxRaw = state.sizeStats.max === -Infinity ? 0 : Math.ceil(state.sizeStats.max || 0); + const minB = Math.max(0, minRaw); + const maxB = Math.max(minB, maxRaw); + + [sizeLo, sizeHi].forEach((inp) => { + inp.min = minB; + inp.max = maxB; + inp.step = 1; + }); + + sizeLo.value = minB; + sizeHi.value = maxB; + sizeLo.style.zIndex = 2; + sizeHi.style.zIndex = 1; + updateSizeUI(false); +} + +function updateSizeUI(triggerRender) { + const { sizeLo, sizeHi, sizeLoVal, sizeHiVal, sizeTrack } = state.ui; + if (+sizeLo.value > +sizeHi.value) { + if (document.activeElement === sizeLo) { + sizeHi.value = sizeLo.value; + } else { + sizeLo.value = sizeHi.value; + } + } + sizeLo.style.zIndex = +sizeLo.value >= +sizeHi.max - 1 ? 4 : 2; + sizeHi.style.zIndex = +sizeHi.value <= +sizeLo.min + 1 ? 3 : 1; + state.filters.sizeLo = +sizeLo.value; + state.filters.sizeHi = +sizeHi.value; + sizeLoVal.textContent = formatSizeLabel(state.filters.sizeLo); + sizeHiVal.textContent = formatSizeLabel(state.filters.sizeHi); + const range = (sizeHi.max - sizeLo.min) || 1; + const minB = +sizeLo.min; + const start = ((state.filters.sizeLo - minB) / range) * 100; + const end = ((state.filters.sizeHi - minB) / range) * 100; + sizeTrack.style.background = `linear-gradient(to right, #e3e7f1 ${start}%, var(--accent) ${start}%, var(--accent) ${end}%, #e3e7f1 ${end}%)`; + if (triggerRender) renderTables(); +} + +function renderTables() { + const backendList = state.backendOrder.filter((env) => state.filters.backends.has(env)); + const frag = document.createDocumentFragment(); + let totalRows = 0; + + for (const test of state.tests) { + const models = filterModels(test.models); + if (!models.length) continue; + totalRows += models.length; + + const block = document.createElement("div"); + block.className = "test-block"; + const heading = document.createElement("h2"); + heading.textContent = test.name; + block.appendChild(heading); + + const tableWrap = document.createElement("div"); + tableWrap.className = "table-wrap"; + const scroller = document.createElement("div"); + scroller.className = "table-scroll"; + + const table = buildSingleTable(models, backendList); + scroller.appendChild(table); + tableWrap.appendChild(scroller); + block.appendChild(tableWrap); + setupResizeOverlay(scroller, backendList, table); + frag.appendChild(block); + } + + state.ui.tables.innerHTML = ""; + if (frag.childNodes.length) { + state.ui.tables.appendChild(frag); + } else { + state.ui.tables.innerHTML = "

No models match the current filters.

"; + } + state.ui.stats.textContent = `Showing ${totalRows.toLocaleString()} model rows across ${backendList.length} configurations`; +} + +function buildSingleTable(models, backendList) { + const table = document.createElement("table"); + const colgroup = document.createElement("colgroup"); + const colModel = document.createElement("col"); + colModel.style.width = `${MODEL_COL_WIDTH}px`; + colgroup.appendChild(colModel); + // Winner colGroup removed + + backendList.forEach((env) => { + const col = document.createElement("col"); + col.style.width = `${state.columnWidths[env] || 200}px`; + col.dataset.env = env; + colgroup.appendChild(col); + }); + table.appendChild(colgroup); + + const thead = document.createElement("thead"); + const headRow = document.createElement("tr"); + headRow.appendChild(makeHeaderCell("Model", "model")); + // Winner header removed + + backendList.forEach((env) => { + const th = makeHeaderCell(env, ""); // REMOVED "backend-header" class + attachHeaderInteractions(th, env); + headRow.appendChild(th); + }); + thead.appendChild(headRow); + table.appendChild(thead); + + const tbody = document.createElement("tbody"); + models.forEach((model) => { + const tr = document.createElement("tr"); + const tdModel = document.createElement("td"); + tdModel.className = "model"; + const head = document.createElement("div"); + head.className = "model-head"; + const nameSpan = document.createElement("span"); + nameSpan.className = "model-name"; + nameSpan.textContent = model.model; + head.appendChild(nameSpan); + tdModel.appendChild(head); + + const meta = document.createElement("div"); + meta.className = "meta"; + meta.textContent = `${model.quant} · ${formatSize(model.sizeB)}`; + tdModel.appendChild(meta); + tr.appendChild(tdModel); + + // Winner cell removed + + backendList.forEach((env) => { + const td = document.createElement("td"); + td.className = "data-cell"; + td.dataset.env = env; + const cell = model.backends[env]; + if (!cell) { + td.innerHTML = `N/A`; + } else if (cell.error || cell.mean == null) { + td.innerHTML = `FAIL`; + } else { + td.innerHTML = `
${cell.mean.toFixed(2)}
`; + } + tr.appendChild(td); + }); + tbody.appendChild(tr); + }); + table.appendChild(tbody); + return table; +} + +function makeHeaderCell(label, extra = "") { + const th = document.createElement("th"); + th.textContent = label; + if (extra) th.className = extra; + return th; +} + +function attachHeaderInteractions(th, env) { + const width = state.columnWidths[env] || 200; + th.style.width = `${width}px`; + th.style.minWidth = `${width}px`; + th.draggable = true; + th.addEventListener("dragstart", (e) => { + state.draggingEnv = env; + th.classList.add("dragging"); + e.dataTransfer.effectAllowed = "move"; + }); + th.addEventListener("dragend", () => { + state.draggingEnv = null; + th.classList.remove("dragging"); + document.querySelectorAll("th.drop-target").forEach((el) => el.classList.remove("drop-target")); + }); + th.addEventListener("dragover", (e) => { + if (!state.draggingEnv || state.draggingEnv === env) return; + e.preventDefault(); + th.classList.add("drop-target"); + }); + th.addEventListener("dragleave", () => th.classList.remove("drop-target")); + th.addEventListener("drop", (e) => { + if (!state.draggingEnv || state.draggingEnv === env) return; + e.preventDefault(); + moveBackend(state.draggingEnv, env); + th.classList.remove("drop-target"); + }); + + const handle = document.createElement("span"); + handle.className = "resize-handle"; + handle.addEventListener("mousedown", (e) => startResize(e, env)); + th.appendChild(handle); +} + +function moveBackend(from, to) { + const order = state.backendOrder; + const fromIdx = order.indexOf(from); + const toIdx = order.indexOf(to); + if (fromIdx === -1 || toIdx === -1) return; + const [col] = order.splice(fromIdx, 1); + order.splice(toIdx, 0, col); + renderBackendList(); + renderTables(); +} + +function filterModels(modelsMap) { + const models = []; + for (const model of modelsMap.values()) { + if (state.filters.search && !model.search_blob.includes(state.filters.search)) continue; + if (state.filters.quant && model.quant !== state.filters.quant) continue; + if (model.sizeB != null) { + if (state.filters.sizeLo != null && model.sizeB < state.filters.sizeLo - 1e-6) continue; + if (state.filters.sizeHi != null && model.sizeB > state.filters.sizeHi + 1e-6) continue; + } + models.push(model); + } + models.sort((a, b) => a.model.localeCompare(b.model)); + return models; +} + +function formatSize(size) { + if (size == null) return "—"; + return `${Number(size).toFixed(1)}B`; +} + +function formatSizeLabel(size) { + if (size >= 1000) return `${(size / 1000).toFixed(1)}kB`; + return `${Math.round(size)}B`; +} + +function startResize(event, env) { + event.preventDefault(); + event.stopPropagation(); + const column = state.columnWidths[env] || 200; + const startX = event.clientX; + const shellRect = state.ui.tables.getBoundingClientRect(); + const guide = document.createElement("div"); + guide.className = "resize-line"; + guide.style.position = "fixed"; + guide.style.top = `${shellRect.top}px`; + guide.style.bottom = `${window.innerHeight - shellRect.bottom}px`; + guide.style.left = `${startX}px`; + guide.style.width = "2px"; + guide.style.background = "var(--accent)"; + guide.style.zIndex = "10"; + document.body.appendChild(guide); + let nextWidth = column; + + const onMove = (e) => { + const delta = e.clientX - startX; + nextWidth = Math.max(80, column + delta); + guide.style.left = `${e.clientX}px`; + }; + + const onUp = () => { + document.removeEventListener("mousemove", onMove); + document.removeEventListener("mouseup", onUp); + guide.remove(); + state.columnWidths[env] = nextWidth; + renderTables(); + }; + + document.addEventListener("mousemove", onMove); + document.addEventListener("mouseup", onUp); +} + +function setupResizeOverlay(tableWrap, backendList, table) { + let overlay = tableWrap.querySelector(".resize-overlay"); + if (!overlay) { + overlay = document.createElement("div"); + overlay.className = "resize-overlay"; + tableWrap.appendChild(overlay); + } else { + overlay.innerHTML = ""; + } + + overlay.style.width = `${tableWrap.clientWidth}px`; + overlay.style.height = `${table.offsetHeight}px`; + + const bars = []; + let offset = MODEL_COL_WIDTH; // Winner column width removed + backendList.forEach((env) => { + const width = state.columnWidths[env] || 200; + const bar = document.createElement("div"); + bar.className = "resize-bar"; + bar.dataset.env = env; + bar.addEventListener("mousedown", (e) => startResize(e, env)); + overlay.appendChild(bar); + bars.push({ bar, offset, width, env }); + offset += width; + }); + + const positionBars = () => { + bars.forEach(({ bar, offset, width }) => { + const left = offset + width - 3 - tableWrap.scrollLeft; + bar.style.left = `${left}px`; + }); + }; + positionBars(); + + if (tableWrap._overlayScroll) { + tableWrap.removeEventListener("scroll", tableWrap._overlayScroll); + } + const onScroll = () => positionBars(); + tableWrap.addEventListener("scroll", onScroll); + tableWrap._overlayScroll = onScroll; + + if (tableWrap._overlayResize) { + tableWrap._overlayResize.disconnect(); + } + const resizeObserver = new ResizeObserver(() => { + overlay.style.width = `${tableWrap.clientWidth}px`; + overlay.style.height = `${table.offsetHeight}px`; + positionBars(); + }); + resizeObserver.observe(tableWrap); + tableWrap._overlayResize = resizeObserver; +} diff --git a/docs/compare.html b/docs/compare.html new file mode 100644 index 0000000..184ce4d --- /dev/null +++ b/docs/compare.html @@ -0,0 +1,804 @@ + + + + + + + GPU Comparison - AMD R9700 vs NVIDIA + + + + + +
+
+

Single GPU Performance Comparison

+

vLLM Decoding Throughput (Tokens/s)

+
Metric: Raw Tokens/s
+

+ ← Back + to Benchmarks +

+
+ +
+
+ + + + + + +
+ + + + + + + +
+ +
+
+ +
+
+ + + + + + + + + +
+
+
+ +
+ * Prices are estimated market averages/MSRP. Power based on TDP.
+ * Composite Score = Weighted index of Perf/$, Perf/Watt, and vRAM/$ (GB per Dollar). Higher is + better.
+ * R9700 (UV+PL): Undervolted -75mV, Power Limit raised to 315W. +
+
+ + + + + \ No newline at end of file diff --git a/docs/comparison_results.json b/docs/comparison_results.json new file mode 100644 index 0000000..426beaf --- /dev/null +++ b/docs/comparison_results.json @@ -0,0 +1,54 @@ +[ + { + "model_name": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI_Qwen3-14B-FP8-dynamic", + "gpus": { + "AMD R9700": 1360.3779166371198, + "AMD R9700 (UV+PL)": 1484.1354103024892, + "NVIDIA RTX 3090": 512.5391321707765, + "NVIDIA RTX 4090": 832.9600726881711, + "NVIDIA RTX 5090": 3251.0147985872827, + "NVIDIA RTX 5000 Ada": 1826.0356413838635, + "NVIDIA A100": 2984.3323175624723 + } + }, + { + "model_name": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "gpus": { + "AMD R9700": 1019.986034633913, + "AMD R9700 (UV+PL)": 1089.8731231693348, + "NVIDIA RTX 3090": 2417.170549559942, + "NVIDIA RTX 4090": 3441.9784198061966, + "NVIDIA RTX 5090": 5550.512371624096, + "NVIDIA RTX 5000 Ada": 2407.9706011923054, + "NVIDIA A100": 3719.016081057382 + } + }, + { + "model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama_Meta-Llama-3.1-8B-Instruct", + "gpus": { + "AMD R9700": 1795.5908237208996, + "AMD R9700 (UV+PL)": 2077.5827013698454, + "NVIDIA RTX 3090": 1830.1667164087519, + "NVIDIA RTX 4090": 2917.0500237249767, + "NVIDIA RTX 5090": 3197.1386468735236, + "NVIDIA RTX 5000 Ada": 2149.8297795990215, + "NVIDIA A100": 3771.8246502543934 + } + }, + { + "model_name": "openai/gpt-oss-20b", + "model_clean": "openai_gpt-oss-20b", + "gpus": { + "AMD R9700": 1095.31715306375, + "AMD R9700 (UV+PL)": 1225.6785068919007, + "NVIDIA RTX 3090": 2066.190202557434, + "NVIDIA RTX 4090": 2685.5060579349843, + "NVIDIA RTX 5090": 6896.704477650824, + "NVIDIA RTX 5000 Ada": 2883.1478405820326, + "NVIDIA A100": 4726.040550232053 + } + } +] \ No newline at end of file diff --git a/docs/generate_comparison_data.py b/docs/generate_comparison_data.py new file mode 100644 index 0000000..7011c96 --- /dev/null +++ b/docs/generate_comparison_data.py @@ -0,0 +1,107 @@ +import json +import re +from pathlib import Path + +# Configuration +SCRIPT_DIR = Path(__file__).parent +PROJECT_ROOT = SCRIPT_DIR.parent +BENCHMARKS_ROOT = PROJECT_ROOT / "benchmarks" +OUTPUT_FILE = SCRIPT_DIR / "comparison_results.json" + +# GPU Mapping +GPU_MAP = { + "benchmark_results_amd-r9700": "AMD R9700", + "benchmark_results_amd-r9700-uv+pl": "AMD R9700 (UV+PL)", + "benchmark_results_nvidia-3090": "NVIDIA RTX 3090", + "benchmark_results_nvidia-4090": "NVIDIA RTX 4090", + "benchmark_results_nvidia-5090": "NVIDIA RTX 5090", + "benchmark_results_nvidia-ada5000": "NVIDIA RTX 5000 Ada", + "benchmark_results_nvidia-a100": "NVIDIA A100" +} + +def parse_model_name(clean_name): + """ + Extracts a pretty display name and basic metadata from the filename-safe model string. + """ + # Example: meta-llama_Meta-Llama-3.1-8B-Instruct -> meta-llama/Meta-Llama-3.1-8B-Instruct + # Example: RedHatAI_Qwen3-14B-FP8-dynamic -> RedHatAI/Qwen3-14B-FP8-dynamic + + # Simple heuristic: replace first underscore with slash to restore Org/Model format + if "_" in clean_name: + display_name = clean_name.replace("_", "/", 1) + else: + display_name = clean_name + + return display_name + +def analyze_benchmarks(): + results = {} + + if not BENCHMARKS_ROOT.exists(): + print(f"Error: {BENCHMARKS_ROOT} not found.") + return + + print(f"Scanning {BENCHMARKS_ROOT}...") + + for folder_name, gpu_display_name in GPU_MAP.items(): + folder_path = BENCHMARKS_ROOT / folder_name + if not folder_path.exists(): + print(f"Warning: {folder_path} not found, skipping.") + continue + + print(f"Processing {gpu_display_name}...") + + # We only care about throughput.json files for "Tokens/s" comparison + # And STRICTLY tp1 (Single GPU) + for json_file in folder_path.glob("*_tp1_throughput.json"): + try: + data = json.loads(json_file.read_text()) + except json.JSONDecodeError: + print(f" Skipping invalid JSON: {json_file.name}") + continue + + # Extract basic info from filename + # Format: {model_clean}_tp1_throughput.json + filename = json_file.name + + # Remove suffix to get model_clean + model_clean = filename.replace("_tp1_throughput.json", "") + + # Get metric + tokens_per_sec = data.get("tokens_per_second", 0) + if not tokens_per_sec: + continue + + # Store in results structure + # Structure: results[model_clean] = { "display_name": "...", "gpus": { "AMD R9700": 123.4, ... } } + + if model_clean not in results: + results[model_clean] = { + "model_name": parse_model_name(model_clean), + "model_clean": model_clean, + "gpus": {} + } + + results[model_clean]["gpus"][gpu_display_name] = tokens_per_sec + + # Convert to list for easier frontend consumption + final_output = [] + for model_key, info in results.items(): + # Only include if we have at least TWO data points for comparison + # (User requested removal of single-GPU only models) + if len(info["gpus"]) < 2: + continue + + final_output.append(info) + + # Sort by model name for consistency + final_output.sort(key=lambda x: x["model_name"]) + + print(f"Found data for {len(final_output)} models.") + + with open(OUTPUT_FILE, "w") as f: + json.dump(final_output, f, indent=2) + print(f"Saved to {OUTPUT_FILE}") + +if __name__ == "__main__": + analyze_benchmarks() diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..1dcc271 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,885 @@ + + + + + + + AMD R9700 vLLM Benchmarks + + + + + +
+
+

AMD R9700 AI PRO vLLM Benchmarks

+

Performance comparison on vLLM

+

+ View on GitHub → + | + + Compare vs NVIDIA & Tuned Results + +

+
+ +
+ + +
+ GPUs: + + +
+ +
+ Attention: + + +
+
+ + + +
+
Loading benchmark results...
+
+ +
+
System Configuration
+
+
+ GPUs + 2x AMD Radeon AI PRO R9700 (64GB Total VRAM) +
+
+ CPU + AMD Ryzen 9 9900X3D +
+
+ Motherboard + ASRock X870E Taichi +
+
+ RAM + 64GB Crucial Pro DDR5 +
+
+ OS + Fedora 43 (6.17.8-300.fc43.x86_64) +
+
+ ROCm + 7.2.53150 (Nightly Build) +
+
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/docs/parse_results.py b/docs/parse_results.py new file mode 100644 index 0000000..60ab5bc --- /dev/null +++ b/docs/parse_results.py @@ -0,0 +1,139 @@ + +import os +import json +import re +from pathlib import Path + +# Config +BENCHMARK_DIR = Path("../benchmarks/benchmark_results_amd-r9700") +OUTPUT_FILE = Path("results.json") + +# Regex to parse model name for quantization and parameters +# Examples: +# "meta-llama/Meta-Llama-3.1-8B-Instruct" +# "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit" +# "RedHatAI/Llama-3.1-8B-Instruct-FP8-block" +PARAMS_REGEX = r"(\d+(?:\.\d+)?)B" +QUANT_REGEX = r"(FP8|AWQ|GPTQ|BF16|4bit|Int4)" + +def extract_meta(model_name): + # Params + params_match = re.search(PARAMS_REGEX, model_name, re.IGNORECASE) + params_b = float(params_match.group(1)) if params_match else None + + # Quant + quant_match = re.search(QUANT_REGEX, model_name, re.IGNORECASE) + quant = quant_match.group(1).upper() if quant_match else "BF16" # Default assumption if no tag? Or unknown. + # Refine quant if 4bit + if quant == "4BIT" or quant == "INT4": + if "GPTQ" in model_name: quant = "GPTQ-4bit" + elif "AWQ" in model_name: quant = "AWQ-4bit" + else: quant = "4-bit" + + return params_b, quant + +def parse_logs(): + runs = [] + + # Define directories and their tags + # (Path, variant_tag) + dirs = [ + (BENCHMARK_DIR, "default"), + (Path("../benchmarks/benchmark_results_amd-r9700-rocm_atten"), "rocm") + ] + + for b_dir, variant in dirs: + if not b_dir.exists(): + print(f"Warning: {b_dir} does not exist, skipping.") + continue + + print(f"Scanning {b_dir} [{variant}]...") + + files = list(b_dir.glob("*.json")) + + for f in files: + fname = f.name + try: + data = json.loads(f.read_text()) + except: + print(f"Skipping bad JSON: {fname}") + continue + + # Infer metadata from filename + parts = fname.split("_tp") + if len(parts) < 2: continue + + model_part = parts[0] + rest = parts[1] + + # TP + tp_match = re.match(r"^(\d+)", rest) + if not tp_match: continue + tp = int(tp_match.group(1)) + + env = f"TP{tp}" + + # Model Name Restoration + if "_" in model_part: + model_display = model_part.replace("_", "/", 1) + else: + model_display = model_part + + params_b, quant = extract_meta(model_display) + + base_run = { + "model": model_display, + "model_clean": model_display, + "env": env, + "variant": variant, + "gpu_config": "dual" if tp > 1 else "single", + "quant": quant, + "params_b": params_b, + "name_params_b": params_b, + "backend": "vLLM", + "error": False + } + + if "throughput" in fname: + tps = data.get("tokens_per_second", 0) + run = base_run.copy() + run["test"] = "Throughput" + run["tps_mean"] = tps + if tps == 0 and "error" in str(data).lower(): + run["error"] = True + runs.append(run) + + elif "latency" in fname: + raw = data.get("raw_output", "") + qps_match = re.search(r"_qps([\d\.]+)_", fname) + qps = qps_match.group(1) if qps_match else "?" + + ttft_m = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw) + ttft = float(ttft_m.group(1)) if ttft_m else 0.0 + + tpot_m = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw) + tpot = float(tpot_m.group(1)) if tpot_m else 0.0 + + # Entry 1: TTFT + r1 = base_run.copy() + r1["test"] = f"TTFT @ QPS {qps}" + r1["tps_mean"] = ttft + runs.append(r1) + + # Entry 2: TPOT + r2 = base_run.copy() + r2["test"] = f"TPOT @ QPS {qps}" + r2["tps_mean"] = tpot + runs.append(r2) + + return runs + +if __name__ == "__main__": + data = {"runs": parse_logs()} + + runs_count = len(data["runs"]) + print(f"Parsed {runs_count} runs.") + + with open(OUTPUT_FILE, "w") as f: + json.dump(data, f, indent=2) + print(f"Written to {OUTPUT_FILE}") diff --git a/docs/results.json b/docs/results.json new file mode 100644 index 0000000..4891a4c --- /dev/null +++ b/docs/results.json @@ -0,0 +1,1404 @@ +{ + "runs": [ + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 186.47 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 59.68 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 129.43 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 61.28 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 57.73 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 25.29 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 14118.91 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 88.1 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 89.11 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 22.0 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1650.9257882903235 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 74.14 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 28.0 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 75.4 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 28.6 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1795.5908237208996 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 120.83 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 45.17 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1325.705122782343 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1878.7081268950026 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "AWQ", + "params_b": 80.0, + "name_params_b": 80.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 680.226281561525 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1360.3779166371198 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 12436.56 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 84.04 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1095.31715306375 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 222.29 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 54.74 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 135.44 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 47.24 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 58.57 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 21.86 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 52928.91 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 61.4 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 984.5314161719857 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 108.91 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 47.9 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1019.986034633913 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 85.92 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 42.81 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "AWQ", + "params_b": 80.0, + "name_params_b": 80.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 72209.01 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "AWQ", + "params_b": 80.0, + "name_params_b": 80.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 71.26 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 104.59 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 38.86 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "AWQ", + "params_b": 80.0, + "name_params_b": 80.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 519.96 + }, + { + "model": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "model_clean": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "AWQ", + "params_b": 80.0, + "name_params_b": 80.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 44.04 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 0.0 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 0.0 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 4078.97 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 78.49 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 80.34 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 37.1 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "default", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 859.7532189051877 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 93.13 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 47.81 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "default", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 2736.811464734795 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 123.92 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 73.89 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 118.15 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 54.56 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 86.17 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 28.21 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 14305.88 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 88.19 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 232.41 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 39.99 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1486.353457790027 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 97.06 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 32.9 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1890.096088342886 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 125.43 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 51.71 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1299.397329054036 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1575.9035072220947 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1429.9541207853842 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 3686.81 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 75.05 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1221.9380837920544 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 37309.58 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 113.18 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 235.33 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 55.13 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 80.83 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 19.52 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 0.0 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 0.0 + }, + { + "model": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-27b-it-FP8-dynamic", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "FP8", + "params_b": 27.0, + "name_params_b": 27.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 621.3330493592667 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 104.14 + }, + { + "model": "RedHatAI/Qwen3-14B-FP8-dynamic", + "model_clean": "RedHatAI/Qwen3-14B-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 14.0, + "name_params_b": 14.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 41.41 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1024.3570195753498 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 81.58 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 37.28 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 127.38 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 45.15 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 1170.97 + }, + { + "model": "openai/gpt-oss-20b", + "model_clean": "openai/gpt-oss-20b", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 20.0, + "name_params_b": 20.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 73.13 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 1.0", + "tps_mean": 74.71 + }, + { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "model_clean": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "BF16", + "params_b": 8.0, + "name_params_b": 8.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 1.0", + "tps_mean": 30.85 + }, + { + "model": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "model_clean": "RedHatAI/gemma-3-12b-it-FP8-dynamic", + "env": "TP1", + "variant": "rocm", + "gpu_config": "single", + "quant": "FP8", + "params_b": 12.0, + "name_params_b": 12.0, + "backend": "vLLM", + "error": false, + "test": "Throughput", + "tps_mean": 1132.6153536237514 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TTFT @ QPS 4.0", + "tps_mean": 194.04 + }, + { + "model": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "model_clean": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", + "env": "TP2", + "variant": "rocm", + "gpu_config": "dual", + "quant": "GPTQ", + "params_b": 30.0, + "name_params_b": 30.0, + "backend": "vLLM", + "error": false, + "test": "TPOT @ QPS 4.0", + "tps_mean": 58.93 + } + ] +} \ No newline at end of file diff --git a/scripts/01-rocm-envs.sh b/scripts/01-rocm-envs.sh new file mode 100644 index 0000000..eada83f --- /dev/null +++ b/scripts/01-rocm-envs.sh @@ -0,0 +1,4 @@ +export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 +export FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE" +export VLLM_TARGET_DEVICE=rocm +export VLLM_USE_TRITON_AWQ=1 \ No newline at end of file diff --git a/scripts/99-toolbox-banner.sh b/scripts/99-toolbox-banner.sh new file mode 100644 index 0000000..67085ea --- /dev/null +++ b/scripts/99-toolbox-banner.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Lightweight banner with machine/GPU and ROCm version (vLLM edition) +# No Triton env sourcing, same info/format as the image/video banner. + +# Only show for interactive shells +case $- in *i*) ;; *) return 0 ;; esac + +oem_info() { + local v="" m="" d lv lm + for d in /sys/class/dmi/id /sys/devices/virtual/dmi/id; do + [[ -r "$d/sys_vendor" ]] && v=$(<"$d/sys_vendor") + [[ -r "$d/product_name" ]] && m=$(<"$d/product_name") + [[ -n "$v" || -n "$m" ]] && break + done + # ARM/SBC fallback + if [[ -z "$v" && -z "$m" && -r /proc/device-tree/model ]]; then + tr -d '\0' /dev/null 2>&1; then + name=$(rocm-smi --showproductname --csv 2>/dev/null | tail -n1 | cut -d, -f2) + [[ -z "$name" ]] && name=$(rocm-smi --showproductname 2>/dev/null | grep -m1 -E 'Product Name|Card series' | sed 's/.*: //') + fi + if [[ -z "$name" ]] && command -v rocminfo >/dev/null 2>&1; then + name=$(rocminfo 2>/dev/null | awk -F': ' '/^[[:space:]]*Name:/{print $2; exit}') + fi + if [[ -z "$name" ]] && command -v lspci >/dev/null 2>&1; then + name=$(lspci -nn 2>/dev/null | grep -Ei 'vga|display|gpu' | grep -i amd | head -n1 | cut -d: -f3-) + fi + # trim + name=$(printf '%s' "$name" | sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+$//' -e 's/[[:space:]]\{2,\}/ /g') + printf '%s\n' "${name:-Unknown AMD GPU}" +} + +rocm_version() { + # Prefer the PyTorch HIP version from the venv, fallback to rocm pkg metadata + local PY="/torch-therock/.venv/bin/python" + [[ -x "$PY" ]] || PY="python" + "$PY" - <<'PY' 2>/dev/null || true +try: + import torch + v = getattr(getattr(torch, "version", None), "hip", "") or "" + if v: + print(v) + else: + raise Exception("no torch.version.hip") +except Exception: + try: + import importlib.metadata as im + try: + print(im.version("_rocm_sdk_core")) + except Exception: + print(im.version("rocm")) + except Exception: + print("") +PY +} + +MACHINE="$(oem_info)" +GPU="$(gpu_name)" +ROCM_VER="$(rocm_version)" + +echo +cat <<'ASCII' + _____ _____ ______ ____ _ _ ___ ______ ___ ___ +| __ \ /\ | __ \| ____/ __ \| \ | | / _ \____ / _ \ / _ \ +| |__) | / \ | | | | |__ | | | | \| | | (_) | / / | | | | | | +| _ / / /\ \ | | | | __|| | | | . ` | \__, | / /| | | | | | | +| | \ \ / ____ \| |__| | |___| |__| | |\ | / / / / | |_| | |_| | +|_| \_\/_/ \_\_____/|______\____/|_| \_| /_/ /_/ \___/ \___/ + + _____ _____ _____ ____ + /\ |_ _| | __ \| __ \ / __ \ + / \ | | | |__) | |__) | | | | + / /\ \ | | | ___/| _ /| | | | + / ____ \ _| |_ | | | | \ \| |__| | + /_/ \_\_____| |_| |_| \_\\____/ + + v L L M +ASCII +echo +printf 'AMD R9700 — vLLM Toolbox (gfx1201, ROCm via TheRock)\n' +[[ -n "$ROCM_VER" ]] && printf 'ROCm nightly: %s\n' "$ROCM_VER" +echo +printf 'Machine: %s\n' "$MACHINE" +printf 'GPU : %s\n\n' "$GPU" +printf 'Repo : https://github.com/kyuz0/amd-r9700-vllm-toolboxes\n' +printf 'Image : docker.io/kyuz0/vllm-therock-gfx1201:latest\n\n' +printf 'Included:\n' +printf ' - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling" +printf ' - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct" +printf ' - %-16s → %s\n' "API test" "curl localhost:8000/v1/chat/completions" +echo +printf 'SSH tip: ssh -L 8000:localhost:8000 user@host\n\n' + +unset PROMPT_COMMAND +PS1='\u@\h:\w\$ ' diff --git a/scripts/generate_models_list.py b/scripts/generate_models_list.py new file mode 100644 index 0000000..aa59332 --- /dev/null +++ b/scripts/generate_models_list.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import sys +import json +import os +from pathlib import Path + +# Add benchmarks dir to path to import config +SCRIPT_DIR = Path(__file__).parent.resolve() +BENCH_DIR = SCRIPT_DIR.parent / "benchmarks" +sys.path.append(str(BENCH_DIR)) + +try: + from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN +except ImportError: + # Fallback if run_vllm_bench not found + MODEL_TABLE = {} + MODELS_TO_RUN = [] + +RESULTS_FILE = BENCH_DIR / "max_context_results.json" + +def get_best_context(model_id, max_tp): + """ + Finds the maximum verified context for the given model + that fits within max_tp (system limit). + """ + if not RESULTS_FILE.exists(): + # Fallback to configured ctx in MODEL_TABLE + return int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)) + + try: + with open(RESULTS_FILE, "r") as f: + data = json.load(f) + except: + return 8192 + + best_ctx = 0 + + # Filter for this model + candidates = [r for r in data if r["model"] == model_id and r["status"] == "success"] + + # Filter by TP <= max_tp (we can't launch TP2 on 1 GPU) + # But we WANT the limit for the Highest Allowable TP. + valid_candidates = [r for r in candidates if r["tp"] <= max_tp] + + if not valid_candidates: + # Fallback to hardcoded table + return int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)) + + # Sort by Context Length (Descending) -> Then TP (Descending) + # This ensures we pick the biggest context possible on the hardware. + valid_candidates.sort(key=lambda x: (x["max_context_1_user"], x["tp"]), reverse=True) + + return valid_candidates[0]["max_context_1_user"] + +def main(): + if len(sys.argv) > 1: + gpu_count = int(sys.argv[1]) + else: + gpu_count = 1 + + for model_id in MODELS_TO_RUN: + config = MODEL_TABLE.get(model_id, {}) + + # 1. Name: Use cleaner name + name = model_id.split("/")[-1] + + # 2. Repo: model_id + + # 3. MaxTP: Min of (Model valid tp max, System GPU Count) + valid_tps = config.get("valid_tp", [1]) + model_max_tp = max(valid_tps) if valid_tps else 1 + + # We cap the reported MaxTP at the system limit for the UI rangebox + # But for finding the context, we look at what is POSSIBLY supported. + # Actually, for the UI, we should only show what is switchable. + ui_max_tp = min(model_max_tp, gpu_count) + + if ui_max_tp < 1: ui_max_tp = 1 # Safety + + # 4. MaxCtx: Get from Results for this UI_MAX_TP + ctx = get_best_context(model_id, ui_max_tp) + + # 5. Flags + flags = [] + if config.get("trust_remote"): flags.append("--trust-remote-code") + if config.get("enforce_eager"): flags.append("--enforce-eager") + flags_str = " ".join(flags) + + # 6. EnvVars + env_dict = config.get("env", {}) + envs_str = " ".join([f"{k}={v}" for k,v in env_dict.items()]) + + # Format: "Name|Repo|MaxCtx|MaxTP|Flags|EnvVars" + print(f"{name}|{model_id}|{ctx}|{ui_max_tp}|{flags_str}|{envs_str}") + +if __name__ == "__main__": + main() diff --git a/scripts/start_vllm.py b/scripts/start_vllm.py new file mode 100644 index 0000000..a33cd1e --- /dev/null +++ b/scripts/start_vllm.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import shutil +import tempfile +import subprocess +from pathlib import Path + +# Add benchmarks dir to path to import config +# Add benchmarks dir to path to import config +SCRIPT_DIR = Path(__file__).parent.resolve() +BENCH_DIR = SCRIPT_DIR.parent / "benchmarks" +OPT_DIR = Path("/opt") + +# Optional environment variable pointing to a local models directory. +# If set, the script will prefer a subfolder under this path matching +# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit) +# when constructing the `vllm serve` command. +LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR") + + +# Check /opt first (Container), then local fallback +if (OPT_DIR / "run_vllm_bench.py").exists(): + sys.path.append(str(OPT_DIR)) +else: + sys.path.append(str(BENCH_DIR)) + +try: + from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN +except ImportError: + print("Error: Could not import run_vllm_bench.py config.") + sys.exit(1) + +if (OPT_DIR / "max_context_results.json").exists(): + RESULTS_FILE = OPT_DIR / "max_context_results.json" +else: + RESULTS_FILE = BENCH_DIR / "max_context_results.json" +HOST = os.getenv("HOST", "0.0.0.0") +PORT = os.getenv("PORT", "8000") + +def check_dependencies(): + if not shutil.which("dialog"): + print("Error: 'dialog' is required. Please install it (apt-get install dialog).") + sys.exit(1) + +def detect_gpus(): + """Detects AMD GPUs via rocm-smi or /dev/dri.""" + try: + # Try rocm-smi first + res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode == 0: + count = res.stdout.count("GPU") + if count > 0: return count + except: pass + + # Fallback to /dev/dri/render* + try: + return len(list(Path("/dev/dri").glob("renderD*"))) + except: + return 1 + +def get_verified_config(model_id, tp_size, max_seqs): + """ + Reads max_context_results.json to find the best verified configuration. + Returns dict: {'ctx': int, 'util': float} + """ + default_config = { + "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)), + "util": 0.90 # Safe default + } + + if not RESULTS_FILE.exists(): + return default_config + + try: + with open(RESULTS_FILE, "r") as f: + data = json.load(f) + + # Filter for Model + TP + Sequences + matches = [r for r in data + if r["model"] == model_id + and r["tp"] == tp_size + and r["max_seqs"] == max_seqs + and r["status"] == "success"] + + if not matches: + # Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context? + # Actually, safer to fallback to default or try finding nearest sequence? + # Let's try finding exact match first. If fail, return default. + return default_config + + # Sort by Util desc, then Context desc + # We prefer higher utilization if available (performance), as long as it is verified success + matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True) + + best = matches[0] + return { + "ctx": best["max_context_1_user"], + "util": float(best["util"]) + } + + except Exception as e: + return default_config + +def run_dialog(args): + """Runs dialog and returns stderr (selection).""" + with tempfile.NamedTemporaryFile(mode="w+") as tf: + cmd = ["dialog"] + args + try: + subprocess.run(cmd, stderr=tf, check=True) + tf.seek(0) + return tf.read().strip() + except subprocess.CalledProcessError: + return None # User cancelled + +def nuke_vllm_cache(): + """Removes vLLM cache directory to fix potential graph/incompatibility issues.""" + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + print(f"Clearing vLLM cache at {cache}...", end="", flush=True) + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + print(" Done.") + time.sleep(1) + except Exception as e: + print(f" Failed: {e}") + +def configure_and_launch(model_idx, gpu_count): + model_id = MODELS_TO_RUN[model_idx] + config = MODEL_TABLE[model_id] + + # Determine whether we have a local copy to serve. Try multiple fallbacks: + # 1) LOCAL_MODEL_DIR// + # 2) LOCAL_MODEL_DIR/ + # 3) case-insensitive match of in LOCAL_MODEL_DIR + model_path = model_id + if LOCAL_MODEL_DIR: + # Full repo path (owner/repo) + candidate_full = os.path.join(LOCAL_MODEL_DIR, model_id) + if os.path.isdir(candidate_full): + model_path = candidate_full + else: + # Repo-name only (last segment) + repo_name = model_id.split('/')[-1] + candidate_repo = os.path.join(LOCAL_MODEL_DIR, repo_name) + if os.path.isdir(candidate_repo): + model_path = candidate_repo + else: + # Fallback: try to find a directory in LOCAL_MODEL_DIR that matches repo_name case-insensitively + try: + for entry in os.listdir(LOCAL_MODEL_DIR): + if entry.lower() == repo_name.lower(): + entry_path = os.path.join(LOCAL_MODEL_DIR, entry) + if os.path.isdir(entry_path): + model_path = entry_path + break + except Exception: + pass ++ # if LOCAL_MODEL_DIR is specified, refuse to fall back to remote ++ if LOCAL_MODEL_DIR and model_path == model_id: ++ print(f"Error: model '{model_id}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}") ++ print("Off‑line mode active; network downloads are disabled.") ++ sys.exit(1) + + # Static Config + valid_tps = config.get("valid_tp", [1]) + max_tp = max(valid_tps) if valid_tps else 1 + + # Defaults + current_tp = min(gpu_count, max_tp) + current_seqs = 1 # Default to 1 concurrent user/request for stability + + # Initial Lookup + verified = get_verified_config(model_id, current_tp, current_seqs) + current_ctx = verified["ctx"] + current_util = verified["util"] + + clear_cache = False + use_eager = config.get("enforce_eager", False) # Default to model config, usually False + use_rocm_attn = False # Default to Triton + + name = model_id.split("/")[-1] + + while True: + cache_status = "YES" if clear_cache else "NO" + eager_status = "YES" if use_eager else "NO" + attn_backend = "ROCm" if use_rocm_attn else "Triton" + + menu_args = [ + "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})", + "--title", f"Configuration: {name}", + "--menu", "Customize Launch Parameters:", "22", "65", "9", + "1", f"Tensor Parallelism: {current_tp}", + "2", f"Concurrent Requests: {current_seqs}", + "3", f"Context Length: {current_ctx} (Verified)", + "4", f"GPU Utilization: {current_util} (Verified)", + "5", f"Attention Backend: {attn_backend}", + "6", f"Erase vLLM Cache: {cache_status}", + "7", f"Force Eager Mode: {eager_status}", + "8", "LAUNCH SERVER" + ] + + choice = run_dialog(menu_args) + if not choice: return False # Back/Cancel + + if choice == "1": + # TP Selection + new_tp = run_dialog([ + "--title", "Tensor Parallelism", + "--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp) + ]) + if new_tp: + new_tp_int = int(new_tp) + if new_tp_int != current_tp: + current_tp = new_tp_int + # RE-CALCULATE Config + verified = get_verified_config(model_id, current_tp, current_seqs) + current_ctx = verified["ctx"] + current_util = verified["util"] + + elif choice == "2": + # Max Seqs Selection + new_seqs = run_dialog([ + "--title", "Concurrent Requests", + "--menu", "Select Max Concurrent Requests:", "12", "40", "4", + "1", "1 (Latency Focus)", + "4", "4 (Balanced)", + "8", "8 (Throughput)", + "16", "16 (Max Load)" + ]) + if new_seqs: + current_seqs = int(new_seqs) + # RE-CALCULATE Config based on new concurrency + verified = get_verified_config(model_id, current_tp, current_seqs) + current_ctx = verified["ctx"] + current_util = verified["util"] + + elif choice == "3": + # Configured Length Override + new_ctx = run_dialog([ + "--title", "Context Length", + "--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx) + ]) + if new_ctx: current_ctx = int(new_ctx) + + elif choice == "4": + # Util Override + pass + + elif choice == "5": + # Toggle Attention Backend + use_rocm_attn = not use_rocm_attn + + elif choice == "6": + # Toggle Cache + if not clear_cache: + # Enabling it -> Show Warning + warn_msg = ( + "WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n" + "This is useful if you are experiencing crashes, 'invalid graph' errors,\n" + "or have switched vLLM versions recently.\n\n" + "However, the next startup will take longer as graphs are re-compiled.\n\n" + "Are you sure you want to enable this?" + ) + confirm = run_dialog([ + "--title", "Erase Cache Warning", + "--yesno", warn_msg, "12", "60" + ]) + + # If confirm is not None (exit 0), it is YES. + if confirm is not None: + clear_cache = True + else: + # Disabling it -> No warning needed + clear_cache = False + + elif choice == "7": + # Toggle Eager Mode + use_eager = not use_eager + + elif choice == "8": + # Launch + break + + # Build Command + subprocess.run(["clear"]) + + if clear_cache: + nuke_vllm_cache() + + cmd = [ + "vllm", "serve", model_path, + "--host", HOST, + "--port", PORT, + "--tensor-parallel-size", str(current_tp), + "--max-num-seqs", str(current_seqs), + "--max-model-len", str(current_ctx), + "--gpu-memory-utilization", str(current_util), + "--dtype", "auto" + ] + + if config.get("trust_remote"): cmd.append("--trust-remote-code") + if use_eager: cmd.append("--enforce-eager") + + # Env Vars + env = os.environ.copy() + env.update(config.get("env", {})) + + if use_rocm_attn: + env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1" + env["VLLM_USE_TRITON_FLASH_ATTN"] = "0" + # Optional: Explicitly mention these in print + + + print("\n" + "="*60) + print(f" Launching: {name}") + if model_path != model_id: + print(f" (using local model at {model_path})") + print(f" Config: TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}") + print(f" Backend: {'ROCm' if use_rocm_attn else 'Triton'}") + if clear_cache: + print(f" Action: Clearing vLLM Cache (~/.cache/vllm)") + print(f" Command: {' '.join(cmd)}") + print("="*60 + "\n") + + os.execvpe("vllm", cmd, env) + +def main(): + check_dependencies() + gpu_count = detect_gpus() + + while True: + # Build Model Menu + menu_items = [] + for i, m_id in enumerate(MODELS_TO_RUN): + name = m_id.split("/")[-1] + # Pre-calc verified ctx for 'default' TP to show in menu? + # Or just show names. Just names is cleaner. + config = MODEL_TABLE[m_id] + menu_items.extend([str(i), name]) + + choice = run_dialog([ + "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})", + "--title", "Select Model", + "--menu", "Choose a model to serve:", "20", "60", "10" + ] + menu_items) + + if not choice: + subprocess.run(["clear"]) + print("Selection cancelled.") + sys.exit(0) + + configure_and_launch(int(choice), gpu_count) + +if __name__ == "__main__": + main() diff --git a/scripts/zz-venv-last.sh b/scripts/zz-venv-last.sh new file mode 100644 index 0000000..eee3228 --- /dev/null +++ b/scripts/zz-venv-last.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Ensure /opt/venv/bin is first even if ~/.local/bin or ~/.cargo/bin prepend themselves via user dotfiles. + +_venv_path_fix() { + # remove any existing /opt/venv/bin entries, then prepend one + local newpath + newpath="$(printf '%s' "$PATH" | awk -v RS=: -v ORS=: '$0!="/opt/venv/bin"{print}')" + PATH="/opt/venv/bin:${newpath%:}" +} + +# run once after shell init; don't duplicate +case "$PROMPT_COMMAND" in + *_venv_path_fix*) : ;; + *) PROMPT_COMMAND="_venv_path_fix${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;; +esac +