cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
SZCJW
2026-03-26 20:50:00 +08:00
commit a04b9ef9c5
208 changed files with 71242 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-and-publish.yml
+98
View File
@@ -0,0 +1,98 @@
name: build-and-publish
on:
workflow_dispatch:
inputs:
tag:
description: "Image tag to publish (e.g. latest)"
required: true
default: "latest"
env:
IMAGE_REPO: kyuz0/vllm-therock-gfx1201
DOCKER_BUILDKIT: "1"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Put Docker on /mnt
run: |
set -eux
echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
sudo systemctl stop docker
sudo rm -rf /var/lib/docker || true
sudo mkdir -p /mnt/docker
sudo systemctl start docker
docker info | grep "Docker Root Dir"
- name: Checkout
uses: actions/checkout@v4
- name: Free disk space
shell: bash
run: |
set -euxo pipefail
echo "Disk BEFORE:"; df -h
sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true
sudo rm -rf /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go || true
docker system prune -af || true
docker builder prune -af || true
sudo apt-get clean
sudo rm -rf /var/lib/apt/lists/*
sudo rm -rf /opt/hostedtoolcache
echo "Disk AFTER:"; df -h
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: BuildKit GC config (8GB cap)
run: |
cat > /tmp/buildkitd.toml <<'EOF'
[worker.oci]
gc = true
gckeepstorage = 8000 # MB
EOF
- name: Set up Buildx (with GC)
uses: docker/setup-buildx-action@v3
with:
buildkitd-flags: --config /tmp/buildkitd.toml
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: docker.io/${{ env.IMAGE_REPO }}
tags: |
type=raw,value=${{ github.event.inputs.tag }}
type=sha
type=raw,value={{date 'YYYYMMDD-HHmmss'}}
labels: |
org.opencontainers.image.source=https://github.com/${{ github.repository }}
org.opencontainers.image.revision=${{ github.sha }}
- name: Build and push
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ./Dockerfile
platforms: linux/amd64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
provenance: false
sbom: false
no-cache: true
- name: Prune buildx cache
if: always()
run: |
docker buildx prune -af --verbose --min-free-space 4gb || true
.gitignore
+1
View File
@@ -0,0 +1 @@
__pycache__
.idea/.gitignore
Generated
+5
View File
@@ -0,0 +1,5 @@
# 默认忽略的文件
/shelf/
/workspace.xml
# 基于编辑器的 HTTP 客户端请求
/httpRequests/
.idea/amd-r9700-vllm-toolboxes.iml
Generated
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.13 (base)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>
.idea/inspectionProfiles/Project_Default.xml
Generated
+14
View File
@@ -0,0 +1,14 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<list>
<option value="python-jose" />
<option value="passlib" />
<option value="aiosqlite" />
</list>
</option>
</inspection_tool>
</profile>
</component>
.idea/inspectionProfiles/profiles_settings.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
.idea/misc.xml
Generated
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="Python 3.13 (base)" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13 (base)" project-jdk-type="Python SDK" />
</project>
.idea/modules.xml
Generated
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/amd-r9700-vllm-toolboxes.iml" filepath="$PROJECT_DIR$/.idea/amd-r9700-vllm-toolboxes.iml" />
</modules>
</component>
</project>
.idea/vcs.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
Dockerfile
+164
View File
@@ -0,0 +1,164 @@
FROM registry.fedoraproject.org/fedora:43
# 1. System Base & Build Tools
# Added 'gperftools-libs' for tcmalloc (fixes double-free)
RUN dnf -y install --setopt=install_weak_deps=False --nodocs \
python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \
gcc gcc-c++ binutils make ffmpeg-free \
cmake ninja-build aria2c tar xz vim nano \
libdrm-devel zlib-devel openssl-devel jq \
numactl-devel gperftools-libs dialog procps-ng \
&& dnf clean all && rm -rf /var/cache/dnf/*
# 2. Install "TheRock" ROCm SDK (Tarball Method)
WORKDIR /tmp
ARG ROCM_MAJOR_VER=7
ARG GFX=gfx120X-all
RUN set -euo pipefail; \
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
| tr '<' '\n' \
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
| sort -V | tail -n1)"; \
echo "Downloading Latest Tarball: ${KEY}"; \
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \
mkdir -p /opt/rocm; \
tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \
rm therock.tar.gz
# 3. Configure Global ROCm Environment
# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash
RUN export ROCM_PATH=/opt/rocm && \
BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
printf '%s\n' \
"export ROCM_PATH=/opt/rocm" \
"export HIP_PLATFORM=amd" \
"export HIP_PATH=/opt/rocm" \
"export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
"export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
"export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
"export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
"export ROCBLAS_USE_HIPBLASLT=1" \
"export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
"export VLLM_TARGET_DEVICE=rocm" \
"export HIP_FORCE_DEV_KERNARG=1" \
"export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
"export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
> /etc/profile.d/rocm-sdk.sh && \
chmod 0644 /etc/profile.d/rocm-sdk.sh
# 4. Python Venv Setup
RUN /usr/bin/python3.13 -m venv /opt/venv
ENV VIRTUAL_ENV=/opt/venv
ENV PATH=/opt/venv/bin:$PATH
ENV PIP_NO_CACHE_DIR=1
RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
# 5. Install PyTorch (TheRock Nightly)
RUN python -m pip install \
--index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
--pre torch torchaudio torchvision
# Flash-Attention
WORKDIR /opt
ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
RUN git clone https://github.com/ROCm/flash-attention.git &&\
cd flash-attention &&\
git checkout main_perf &&\
python setup.py install && \
cd /opt && rm -rf /opt/flash-attention
# 6. Clone vLLM
RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
WORKDIR /opt/vllm
# --- PATCHING ---
# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers),
# vLLM falls back to CPU. We patch it to force ROCm detection.
RUN echo "import sys, re" > patch_vllm.py && \
echo "from pathlib import Path" >> patch_vllm.py && \
# Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks
echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
# Patch 2: rocm.py - Mock amdsmi and force device name
echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
echo "txt = header + txt" >> patch_vllm.py && \
echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
python patch_vllm.py
# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
ENV ROCM_HOME="/opt/rocm"
ENV HIP_PATH="/opt/rocm"
ENV VLLM_TARGET_DEVICE="rocm"
ENV PYTORCH_ROCM_ARCH="gfx1201"
ENV HIP_ARCHITECTURES="gfx1201"
ENV AMDGPU_TARGETS="gfx1201"
ENV MAX_JOBS="4"
# --- FIX FOR SEGFAULT ---
# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC.
# This aligns the ABI of the compiled vLLM extensions with PyTorch.
ENV CC="/opt/rocm/llvm/bin/clang"
ENV CXX="/opt/rocm/llvm/bin/clang++"
RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
python -m pip install /tmp/dist/*.whl
# --- bitsandbytes (ROCm) ---
WORKDIR /opt
RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
WORKDIR /opt/bitsandbytes
# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
ENV HIP_PLATFORM="amd"
ENV CMAKE_PREFIX_PATH="/opt/rocm"
# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
RUN cmake -S . \
-DGPU_TARGETS="gfx1201" \
-DBNB_ROCM_ARCH="gfx1201" \
-DCOMPUTE_BACKEND=hip \
-DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
&& \
make -j$(nproc) && \
python -m pip install --no-cache-dir . --no-build-isolation --no-deps
# 8. Final Cleanup & Runtime
WORKDIR /opt
RUN chmod -R a+rwX /opt && \
find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
rm -rf /root/.cache/pip || true && \
dnf clean all && rm -rf /var/cache/dnf/*
COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
COPY scripts/start_vllm.py /usr/local/bin/start-vllm
COPY benchmarks/max_context_results.json /opt/max_context_results.json
COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
CMD ["/bin/bash"]
README.md
+209
View File
@@ -0,0 +1,209 @@
# AMD Radeon 9700 AI PRO (gfx1201) — vLLM Toolbox/Container
An **fedora-based** Docker/Podman container that is **Toolbx-compatible** (usable as a Fedora toolbox) for serving LLMs with **vLLM** on **AMD Radeon R9700 (gfx1201)**. Built on the TheRock nightly builds for ROCM.
![Demo](demo.gif)
---
## Table of Contents
* [Tested Models (Benchmarks)](#tested-models-benchmarks)
* [1) Toolbx vs Docker/Podman](#1-toolbx-vs-dockerpodman)
* [2) Quickstart — Fedora Toolbx](#2-quickstart--fedora-toolbx)
* [3) Quickstart — Ubuntu (Distrobox)](#3-quickstart--ubuntu-distrobox)
* [4) Testing the API](#4-testing-the-api)
* [5) Use a Web UI for Chatting](#5-use-a-web-ui-for-chatting)
## Tested Models (Benchmarks)
View full benchmarks at: [https://kyuz0.github.io/amd-r9700-vllm-toolboxes/](https://kyuz0.github.io/amd-r9700-vllm-toolboxes/)
*Run benchmarks now include a comparison between the default Triton backend and the optional ROCm attention backend.*
**Table Key:** Cell values represent `Max Context Length (GPU Memory Utilization)`.
| Model | TP | 1 Req | 4 Reqs | 8 Reqs | 16 Reqs |
| :--- | :--- | :--- | :--- | :--- | :--- |
| **`meta-llama/Meta-Llama-3.1-8B-Instruct`** | 1 | 127k (0.98) | 127k (0.98) | 127k (0.98) | 127k (0.98) |
| | 2 | 105k (0.98) | 105k (0.98) | 105k (0.98) | 105k (0.98) |
| **`openai/gpt-oss-20b`** | 1 | 131k (0.98) | 131k (0.98) | 131k (0.98) | 131k (0.98) |
| | 2 | 131k (0.95) | 131k (0.95) | 131k (0.95) | 131k (0.95) |
| **`RedHatAI/Qwen3-14B-FP8-dynamic`** | 1 | 41k (0.98) | 41k (0.98) | 41k (0.98) | 41k (0.98) |
| | 2 | 41k (0.95) | 41k (0.95) | 41k (0.95) | 41k (0.95) |
| **`cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit`** | 1 | 151k (0.98) | 151k (0.98) | 151k (0.98) | 151k (0.98) |
| | 2 | 262k (0.98) | 262k (0.98) | 262k (0.98) | 262k (0.98) |
| **`cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit`** | 2 | 156k (0.98) | 156k (0.98) | 156k (0.98) | 156k (0.98) |
| **`RedHatAI/gemma-3-12b-it-FP8-dynamic`** | 1 | 45k (0.98) | 45k (0.98) | 45k (0.98) | 45k (0.98) |
| | 2 | 126k (0.98) | 126k (0.98) | 121k (0.95) | 121k (0.95) |
| **`RedHatAI/gemma-3-27b-it-FP8-dynamic`** | 2 | 60k (0.98) | 60k (0.98) | 60k (0.98) | 60k (0.98) |
### Advanced Tuning
See [TUNING.md](TUNING.md) for a guide on how to enable undervolting and raise the power limit on AMD R9700 cards on Linux to improve performance and efficiency.
### 🆕 Update: Comparison of Attention Backends (Triton vs ROCm)
*Added Support for ROCm Native Attention Backend*
I have added the ability to switch between the default **Triton** backend and the experimental **ROCm** native backend for attention operations. This provides you with more flexibility to optimize for stability or throughput depending on your specific model and workload.
| Backend | Stability | Throughput | Compatibility |
| :--- | :--- | :--- | :--- |
| **Triton** (Default) | ✅ **High** | 🔸 Good | Works with all tested models |
| **ROCm** | ⚠️ **Experimental** | 🚀 **Highest** | May fail with complex architectures |
**Key Differences:**
- **Triton**: The safe choice. It uses the Triton compiler to generate kernels and is the standard for vLLM on AMD.
- **ROCm**: Uses composable kernel based attention. In my benchmarks, this often yields higher throughput (tokens/sec) but can be less stable, leading to crashes or "invalid graph" errors on some newer models.
**How to Use:**
1. **Easy Mode**: Select the backend in the `start-vllm` wizard (Item 5 in the menu).
2. **Manual Mode**: Export the following environment variables before running `vllm serve`:
```bash
export VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1
export VLLM_USE_TRITON_FLASH_ATTN=0
```
---
## 1) Toolbx vs Docker/Podman
The `kyuz0/vllm-therock-gfx1201:latest` image can be used both as: 
* **Fedora Toolbx (recommended for development):** Toolbx shares your **HOME** and user, so models/configs live on the host. Great for iterating quickly while keeping the host clean.
* **Docker/Podman (recommended for deployment/perf):** Use for running vLLM as a service (host networking, IPC tuning, etc.). Always **mount a host directory** for model weights so they stay outside the container.
---
## 2) Quickstart — Fedora Toolbx
Create a toolbox that exposes the GPU and relaxes seccomp to avoid ROCm syscall issues:
```bash
toolbox create vllm-r9700 \
--image docker.io/kyuz0/vllm-therock-gfx1201:latest \
-- --device /dev/dri --device /dev/kfd \
--group-add video --group-add render --security-opt seccomp=unconfined
```
Enter it:
```bash
toolbox enter vllm-r9700
```
**Model storage:** Models are downloaded to `~/.cache/huggingface` by default. This directory is shared with the host if you created the toolbox correctly, so downloads persist.
### Serving a Model (Easiest Way)
The toolbox includes a TUI wizard called **`start-vllm`** which includes pre-configured models and handles launch flags. It also allows you to select the experimental **ROCm attention backend**. This is the easiest way to get started.
```bash
# if your weights live on disk instead of on HuggingFace, point the
# launcher at the directory that contains the model folders. the
# script will look for a subdirectory matching the repo ID and use it
# when launching.
export LOCAL_MODEL_DIR=/workspace/models
start-vllm
# you can also just run the CLI yourself and pass the path directly:
vllm serve /workspace/models/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit ...
```
> **Cache note:** vLLM writes compiled kernels to `~/.cache/vllm/`.
---
### 离线/本地模式
如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。
脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**，
不会尝试下载任何内容。适用于没有外网或受限环境的部署。
```bash
export LOCAL_MODEL_DIR=/workspace/models
start-vllm # 本地模式，如果模型缺失则失败
```
### 离线/本地模式
如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。
脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**，
不会尝试下载任何内容。适用于没有外网或受限环境的部署。
```bash
export LOCAL_MODEL_DIR=/workspace/models
start-vllm # 本地模式，如果模型缺失则失败
```
## 3) Quickstart — Ubuntu (Distrobox)
Ubuntu’s toolbox package still breaks GPU access, so use Distrobox instead:
```bash
distrobox create -n vllm-r9700 \
--image docker.io/kyuz0/vllm-therock-gfx1201:latest \
--additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined"
distrobox enter vllm-r9700
```
> **Verification:** Run `rocm-smi` to check GPU status.
### Serving a Model
Same as above, you can use the **`start-vllm`** wizard to launch models easily.
```bash
start-vllm
```
---
## 4) Testing the API
Once the server is up, hit the OpenAI‑compatible endpoint:
```bash
curl -X POST http://localhost:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello! Test the performance."}]}'
```
You should receive a JSON response with a `choices[0].message.content` reply.
If you don't want to bother specifying the model name, you can run this which will query the currently deployed model:
```bash
MODEL=$(curl -s http://localhost:8000/v1/models | jq -r '.data[0].id') curl -X POST http://localhost:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d "{
\"model\": \"$MODEL\",
\"messages\":[{\"role\":\"user\",\"content\":\"Hello! Test the performance.\"}]
}"
```
---
## 5) Use a Web UI for Chatting
If vLLM is on a remote server, expose port 8000 via SSH port forwarding:
```bash
ssh -L 0.0.0.0:8000:localhost:8000 <vllm-host>
```
Then, you can start HuggingFace ChatUI like this (on your host):
```bash
docker run -p 3000:3000 \
--add-host=host.docker.internal:host-gateway \
-e OPENAI_BASE_URL=http://host.docker.internal:8000/v1 \
-e OPENAI_API_KEY=dummy \
-v chat-ui-data:/data \
ghcr.io/huggingface/chat-ui-db
```
TUNING.md
+108
View File
@@ -0,0 +1,108 @@
# AMD Radeon 9700 AI PRO (Navi 48) Tuning Guide
This guide show you how to enable undervolting and raise the power limit on the Radeon PRO R9700 on Fedora 43.
## 1. Prerequisites (Enable Overclocking)
The `ppfeaturemask` must be set to unlock voltage control.
```bash
sudo grubby --update-kernel=ALL --args="amdgpu.ppfeaturemask=0xffffffff"
```
**Reboot your system after running this.**
## 2. Identify Your GPU
You need the **PCI Bus ID** of the target card (e.g., `07:00.0`).
Run this command to list your AMD GPUs:
```bash
lspci -nn | grep "VGA" | grep "AMD"
```
*Example output:*
`07:00.0 VGA compatible controller...`
*(In this example, your ID is `0000:07:00.0`)*
## 3. Tuning Script
Save this script as `tune_r9700.sh`.
**Important:** Edit the `PCI_ID` variable at the top to match the ID you found in Step 2.
```bash
#!/bin/bash
# --- CONFIGURATION ---
# Replace this with your specific PCI ID from 'lspci'
# Format must be 0000:XX:XX.X
PCI_ID="0000:07:00.0"
# ---------------------
# 1. Robustly Find the Card Name (e.g., card1) directly from the PCI Bus
# We look inside the PCI device's 'drm' folder for a folder starting with 'card' followed only by numbers.
if [ ! -d "/sys/bus/pci/devices/$PCI_ID/drm" ]; then
echo "Error: PCI Device $PCI_ID not found or has no DRM driver attached."
exit 1
fi
# This finds 'card1' but ignores 'card1-DP-6'
CARD_NAME=$(ls "/sys/bus/pci/devices/$PCI_ID/drm" | grep -E '^card[0-9]+$' | head -n 1)
if [ -z "$CARD_NAME" ]; then
echo "Error: Could not determine card name for $PCI_ID"
exit 1
fi
# Construct the clean path (e.g., /sys/class/drm/card1)
CARD_PATH="/sys/class/drm/$CARD_NAME"
echo "Tuning GPU: $CARD_NAME (at $CARD_PATH)..."
# 2. Force Manual Performance Level (Required for UV)
# We use 'tee' without pipe to ensure exact errors are caught, but your 'echo | sudo tee' method is fine.
echo "manual" | sudo tee "$CARD_PATH/device/power_dpm_force_performance_level" > /dev/null
if [ $? -ne 0 ]; then echo "Failed to set Manual mode. Check permissions/path."; exit 1; fi
# 3. Apply -75mV Undervolt
echo "vo -75" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null
echo "c" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null
echo "Applied Undervolt (-75mV)"
# 4. Set Power Limit to 315W
# Find the hwmon directory strictly inside the device
HWMON_DIR=$(find "$CARD_PATH/device/hwmon/" -maxdepth 1 -name "hwmon*" | head -n 1)
if [ -n "$HWMON_DIR" ]; then
echo "315000000" | sudo tee "$HWMON_DIR/power1_cap" > /dev/null
echo "Applied Power Limit (315W)"
else
echo "Error: Could not find hwmon directory for power limit."
fi
```
### Usage
```bash
chmod +x tune_r9700.sh
sudo ./tune_r9700.sh
```
## 4\. Verification
Run these commands to confirm settings are active.
**Check Undervolt:**
```bash
# Look for: OD_VDDGFX_OFFSET: -75mV
cat /sys/class/drm/card0/device/pp_od_clk_voltage
```
**Check Power Limit:**
```bash
# Should be ~315 W
cat /sys/class/drm/card0/device/hwmon/hwmon*/power1_cap
```
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 14:59:00 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc7c8a23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ce1f54c0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.72 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.76 \nPeak output token throughput (tok/s): 344.00 \nPeak concurrent requests: 16.00 \nTotal token throughput (tok/s): 391.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.14 \nMedian TTFT (ms): 87.12 \nP99 TTFT (ms): 220.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.41 \nMedian TPOT (ms): 41.23 \nP99 TPOT (ms): 46.01 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.43 \nMedian ITL (ms): 40.25 \nP99 ITL (ms): 86.38 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:02:31 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f987a537740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-959ca2a7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 668 \nFailed requests: 52 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 180.01 \nTotal input tokens: 137901 \nTotal generated tokens: 128399 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 713.31 \nPeak output token throughput (tok/s): 1215.00 \nPeak concurrent requests: 72.00 \nTotal token throughput (tok/s): 1479.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 123.92 \nMedian TTFT (ms): 95.36 \nP99 TTFT (ms): 403.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.89 \nMedian TPOT (ms): 55.38 \nP99 TPOT (ms): 442.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.97 \nMedian ITL (ms): 49.28 \nP99 ITL (ms): 187.29 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log
+16815
View File
File diff suppressed because one or more lines are too long
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 518.4320176600013,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 1.9288932124863885,
"tokens_per_second": 1429.9541207853842
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_server.log
+432
View File
@@ -0,0 +1,432 @@
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
WARNING 12-19 17:09:03 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
(APIServer pid=75372) INFO 12-19 17:09:03 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
(APIServer pid=75372) INFO 12-19 17:09:03 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32000, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
(APIServer pid=75372) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(APIServer pid=75372) INFO 12-19 17:09:08 [model.py:514] Resolved architecture: Qwen3ForCausalLM
(APIServer pid=75372) INFO 12-19 17:09:08 [model.py:1636] Using max model len 32000
(APIServer pid=75372) INFO 12-19 17:09:08 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
(EngineCore_DP0 pid=75534) INFO 12-19 17:09:12 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32000, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
(EngineCore_DP0 pid=75534) WARNING 12-19 17:09:12 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
INFO 12-19 17:09:16 [pynccl.py:111] vLLM is using nccl==2.27.3
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
(Worker_TP0 pid=75616) INFO 12-19 17:09:17 [gpu_model_runner.py:3562] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic...
(Worker_TP1 pid=75617) INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP0 pid=75616) INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.57it/s]
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.02it/s]
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.14s/it]
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.19s/it]
(Worker_TP0 pid=75616)
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.09s/it]
(Worker_TP0 pid=75616)
(Worker_TP0 pid=75616) INFO 12-19 17:09:22 [default_loader.py:308] Loading weights took 4.43 seconds
(Worker_TP0 pid=75616) INFO 12-19 17:09:23 [gpu_model_runner.py:3659] Model loading took 7.8555 GiB memory and 5.221202 seconds
(Worker_TP0 pid=75616) INFO 12-19 17:09:28 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/8760cb82b9/rank_0_0/backbone for vLLM's torch.compile
(Worker_TP0 pid=75616) INFO 12-19 17:09:28 [backends.py:694] Dynamo bytecode transform time: 5.45 s
(Worker_TP1 pid=75617) INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP0 pid=75616) INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP0 pid=75616) INFO 12-19 17:09:36 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 4.77 s
(Worker_TP0 pid=75616) INFO 12-19 17:09:36 [monitor.py:34] torch.compile takes 10.22 s in total
(Worker_TP0 pid=75616) INFO 12-19 17:09:39 [gpu_worker.py:375] Available KV cache memory: 22.49 GiB
(EngineCore_DP0 pid=75534) INFO 12-19 17:09:39 [kv_cache_utils.py:1291] GPU KV cache size: 294,752 tokens
(EngineCore_DP0 pid=75534) INFO 12-19 17:09:39 [kv_cache_utils.py:1296] Maximum concurrency for 32,000 tokens per request: 9.21x
(Worker_TP0 pid=75616)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:08, 2.07it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:07, 2.15it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:07, 2.27it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:06, 2.32it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:05, 2.36it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:05, 2.39it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:02<00:04, 2.42it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:04, 2.44it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:03<00:04, 2.47it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:03, 2.48it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:04<00:03, 2.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:04<00:02, 2.50it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:05<00:02, 2.50it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:05<00:02, 2.50it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:06<00:01, 2.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:06<00:01, 2.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:06<00:00, 2.49it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:07<00:00, 2.48it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.51it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.44it/s]
(Worker_TP0 pid=75616)
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:06, 1.63it/s]
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:01<00:04, 1.99it/s]
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.20it/s]
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.30it/s]
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.36it/s]
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.40it/s]
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.44it/s]
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.47it/s]
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.50it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.51it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.53it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.39it/s]
(Worker_TP0 pid=75616) INFO 12-19 17:09:52 [gpu_model_runner.py:4610] Graph capturing finished in 13 secs, took 0.66 GiB
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
(Worker_TP1 pid=75617) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
(Worker_TP0 pid=75616) ERROR 12-19 17:09:52 [multiproc_executor.py:826]
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] EngineCore failed to start.
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] Traceback (most recent call last):
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] super().__init__(
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] )
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] vllm_config
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] )
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] return aggregate(get_response())
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~^^
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] raise RuntimeError(
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] ...<2 lines>...
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] )
(EngineCore_DP0 pid=75534) ERROR 12-19 17:09:52 [core.py:866] RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
(EngineCore_DP0 pid=75534) Process EngineCore_DP0:
(EngineCore_DP0 pid=75534) Traceback (most recent call last):
(EngineCore_DP0 pid=75534) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
(EngineCore_DP0 pid=75534) self.run()
(EngineCore_DP0 pid=75534) ~~~~~~~~^^
(EngineCore_DP0 pid=75534) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=75534) self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
(EngineCore_DP0 pid=75534) raise e
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
(EngineCore_DP0 pid=75534) engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
(EngineCore_DP0 pid=75534) super().__init__(
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=75534) vllm_config, executor_class, log_stats, executor_fail_callback
(EngineCore_DP0 pid=75534) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) )
(EngineCore_DP0 pid=75534) ^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=75534) num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=75534) vllm_config
(EngineCore_DP0 pid=75534) ^^^^^^^^^^^
(EngineCore_DP0 pid=75534) )
(EngineCore_DP0 pid=75534) ^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
(EngineCore_DP0 pid=75534) self.model_executor.initialize_from_config(kv_cache_configs)
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
(EngineCore_DP0 pid=75534) self.collective_rpc("compile_or_warm_up_model")
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
(EngineCore_DP0 pid=75534) return aggregate(get_response())
(EngineCore_DP0 pid=75534) ~~~~~~~~~~~~^^
(EngineCore_DP0 pid=75534) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
(EngineCore_DP0 pid=75534) raise RuntimeError(
(EngineCore_DP0 pid=75534) ...<2 lines>...
(EngineCore_DP0 pid=75534) )
(EngineCore_DP0 pid=75534) RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
(Worker_TP1 pid=75617) INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
(Worker_TP0 pid=75616) INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
(APIServer pid=75372) Traceback (most recent call last):
(APIServer pid=75372) File "/opt/venv/bin/vllm", line 7, in <module>
(APIServer pid=75372) sys.exit(main())
(APIServer pid=75372) ~~~~^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
(APIServer pid=75372) args.dispatch_function(args)
(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
(APIServer pid=75372) uvloop.run(run_server(args))
(APIServer pid=75372) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
(APIServer pid=75372) return __asyncio.run(
(APIServer pid=75372) ~~~~~~~~~~~~~^
(APIServer pid=75372) wrapper(),
(APIServer pid=75372) ^^^^^^^^^^
(APIServer pid=75372) ...<2 lines>...
(APIServer pid=75372) **run_kwargs
(APIServer pid=75372) ^^^^^^^^^^^^
(APIServer pid=75372) )
(APIServer pid=75372) ^
(APIServer pid=75372) File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
(APIServer pid=75372) return runner.run(main)
(APIServer pid=75372) ~~~~~~~~~~^^^^^^
(APIServer pid=75372) File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
(APIServer pid=75372) return self._loop.run_until_complete(task)
(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
(APIServer pid=75372) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
(APIServer pid=75372) return await main
(APIServer pid=75372) ^^^^^^^^^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
(APIServer pid=75372) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
(APIServer pid=75372) async with build_async_engine_client(
(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=75372) args,
(APIServer pid=75372) ^^^^^
(APIServer pid=75372) client_config=client_config,
(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=75372) ) as engine_client:
(APIServer pid=75372) ^
(APIServer pid=75372) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
(APIServer pid=75372) return await anext(self.gen)
(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
(APIServer pid=75372) async with build_async_engine_client_from_engine_args(
(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=75372) engine_args,
(APIServer pid=75372) ^^^^^^^^^^^^
(APIServer pid=75372) ...<2 lines>...
(APIServer pid=75372) client_config=client_config,
(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=75372) ) as engine:
(APIServer pid=75372) ^
(APIServer pid=75372) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
(APIServer pid=75372) return await anext(self.gen)
(APIServer pid=75372) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
(APIServer pid=75372) async_llm = AsyncLLM.from_vllm_config(
(APIServer pid=75372) vllm_config=vllm_config,
(APIServer pid=75372) ...<6 lines>...
(APIServer pid=75372) client_index=client_index,
(APIServer pid=75372) )
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
(APIServer pid=75372) return cls(
(APIServer pid=75372) vllm_config=vllm_config,
(APIServer pid=75372) ...<9 lines>...
(APIServer pid=75372) client_index=client_index,
(APIServer pid=75372) )
(APIServer pid=75372) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
(APIServer pid=75372) self.engine_core = EngineCoreClient.make_async_mp_client(
(APIServer pid=75372) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_Qwen3-14B-FP8-dynamic_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 470.41839592500037,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 2.1257672077931065,
"tokens_per_second": 1575.9035072220947
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:31:43 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3319c47740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-242c7681-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.10 \nTotal input tokens: 40429 \nTotal generated tokens: 38744 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 191.71 \nPeak output token throughput (tok/s): 354.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 391.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 127.38 \nMedian TTFT (ms): 107.28 \nP99 TTFT (ms): 286.14 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.15 \nMedian TPOT (ms): 45.01 \nP99 TPOT (ms): 56.92 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.71 \nMedian ITL (ms): 42.72 \nP99 ITL (ms): 114.80 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:35:15 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f567795b740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-8a860b01-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 224.44 \nTotal input tokens: 158086 \nTotal generated tokens: 153768 \nRequest throughput (req/s): 3.21 \nOutput token throughput (tok/s): 685.11 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 111.00 \nTotal token throughput (tok/s): 1389.47 \n---------------Time to First Token----------------\nMean TTFT (ms): 3686.81 \nMedian TTFT (ms): 3310.63 \nP99 TTFT (ms): 10615.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 75.05 \nMedian TPOT (ms): 75.32 \nP99 TPOT (ms): 105.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 74.21 \nMedian ITL (ms): 62.11 \nP99 ITL (ms): 246.46 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log
+1031
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 666.9801866829985,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 1.499294911552266,
"tokens_per_second": 1132.6153536237514
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 18:04:12 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb4e3457740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b978cded-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.92 \nTotal input tokens: 40429 \nTotal generated tokens: 38727 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 196.66 \nPeak output token throughput (tok/s): 393.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 401.97 \n---------------Time to First Token----------------\nMean TTFT (ms): 232.41 \nMedian TTFT (ms): 152.06 \nP99 TTFT (ms): 582.73 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 39.99 \nMedian TPOT (ms): 38.66 \nP99 TPOT (ms): 75.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.94 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 310.76 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 18:07:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3b12e0f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3d8d3c55-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 299.13 \nTotal input tokens: 158086 \nTotal generated tokens: 152394 \nRequest throughput (req/s): 2.41 \nOutput token throughput (tok/s): 509.46 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 294.00 \nTotal token throughput (tok/s): 1037.94 \n---------------Time to First Token----------------\nMean TTFT (ms): 37309.58 \nMedian TTFT (ms): 42371.37 \nP99 TTFT (ms): 86881.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 113.18 \nMedian TPOT (ms): 113.38 \nP99 TPOT (ms): 240.19 \n---------------Inter-token Latency----------------\nMean ITL (ms): 109.51 \nMedian ITL (ms): 58.62 \nP99 ITL (ms): 472.97 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log
+1053
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 581.3710580350016,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 1.72007186491178,
"tokens_per_second": 1299.397329054036
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 17:37:17 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f2ac681f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0a333e52-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 45 \nFailed requests: 135 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 180.00 \nTotal input tokens: 10566 \nTotal generated tokens: 7279 \nRequest throughput (req/s): 0.25 \nOutput token throughput (tok/s): 40.44 \nPeak output token throughput (tok/s): 234.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 99.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 235.33 \nMedian TTFT (ms): 121.50 \nP99 TTFT (ms): 734.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 55.13 \nMedian TPOT (ms): 54.64 \nP99 TPOT (ms): 75.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.59 \nMedian ITL (ms): 50.79 \nP99 ITL (ms): 355.12 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": false,
"raw_output": "WARNING 12-19 17:40:29 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7fcc91f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-660ac132-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log
+2570
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 1215.8245900149996,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 0.8224870661545537,
"tokens_per_second": 621.3330493592667
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:19:35 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4356b0b740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb7743a2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.40 \nTotal input tokens: 38358 \nTotal generated tokens: 40237 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.79 \nPeak output token throughput (tok/s): 315.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 392.20 \n---------------Time to First Token----------------\nMean TTFT (ms): 125.43 \nMedian TTFT (ms): 115.75 \nP99 TTFT (ms): 233.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 51.71 \nMedian TPOT (ms): 52.62 \nP99 TPOT (ms): 60.97 \n---------------Inter-token Latency----------------\nMean ITL (ms): 51.22 \nMedian ITL (ms): 51.73 \nP99 ITL (ms): 115.89 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:23:04 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8a19abf740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-783b8bf2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.51 \nTotal input tokens: 146694 \nTotal generated tokens: 152753 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 612.20 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 171.00 \nTotal token throughput (tok/s): 1200.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 14305.88 \nMedian TTFT (ms): 18966.25 \nP99 TTFT (ms): 31290.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.19 \nMedian TPOT (ms): 90.39 \nP99 TPOT (ms): 108.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.87 \nMedian ITL (ms): 84.16 \nP99 ITL (ms): 193.01 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log
+1038
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 723.7066626509986,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 1.3817753125788776,
"tokens_per_second": 1024.3570195753498
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 16:03:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fbdd8daf740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2904e4f8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.44 \nTotal input tokens: 38358 \nTotal generated tokens: 39896 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 207.32 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal token throughput (tok/s): 406.64 \n---------------Time to First Token----------------\nMean TTFT (ms): 97.06 \nMedian TTFT (ms): 80.47 \nP99 TTFT (ms): 208.68 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 32.90 \nMedian TPOT (ms): 32.05 \nP99 TPOT (ms): 41.21 \n---------------Inter-token Latency----------------\nMean ITL (ms): 32.45 \nMedian ITL (ms): 30.18 \nP99 ITL (ms): 85.86 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 16:53:26 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f094443f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c9db55f5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 206.61 \nTotal input tokens: 146694 \nTotal generated tokens: 153351 \nRequest throughput (req/s): 3.48 \nOutput token throughput (tok/s): 742.22 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 76.00 \nTotal token throughput (tok/s): 1452.22 \n---------------Time to First Token----------------\nMean TTFT (ms): 194.04 \nMedian TTFT (ms): 127.11 \nP99 TTFT (ms): 1506.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 58.93 \nMedian TPOT (ms): 59.18 \nP99 TPOT (ms): 82.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 58.36 \nMedian ITL (ms): 52.92 \nP99 ITL (ms): 178.94 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log
+846
View File
@@ -0,0 +1,846 @@
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
WARNING 12-19 16:52:23 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
(APIServer pid=69653) INFO 12-19 16:52:23 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
(APIServer pid=69653) INFO 12-19 16:52:23 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
(APIServer pid=69653) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(APIServer pid=69653) INFO 12-19 16:52:28 [model.py:514] Resolved architecture: Qwen3MoeForCausalLM
(APIServer pid=69653) INFO 12-19 16:52:28 [model.py:1636] Using max model len 24576
(APIServer pid=69653) INFO 12-19 16:52:28 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
(EngineCore_DP0 pid=69815) INFO 12-19 16:52:32 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
(EngineCore_DP0 pid=69815) WARNING 12-19 16:52:32 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
INFO 12-19 16:52:35 [pynccl.py:111] vLLM is using nccl==2.27.3
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
(Worker_TP0 pid=69898) INFO 12-19 16:52:36 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit...
(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
(Worker_TP1 pid=69899) INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
(Worker_TP1 pid=69899) WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
(Worker_TP0 pid=69898) INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
(Worker_TP0 pid=69898) WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 1.61it/s]
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:03, 1.60s/it]
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:05<00:01, 1.90s/it]
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 2.13s/it]
(Worker_TP0 pid=69898)
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 1.91s/it]
(Worker_TP0 pid=69898)
(Worker_TP0 pid=69898) INFO 12-19 16:52:46 [default_loader.py:308] Loading weights took 7.71 seconds
(Worker_TP0 pid=69898) INFO 12-19 16:52:46 [gpu_model_runner.py:3659] Model loading took 8.1992 GiB memory and 9.032124 seconds
(Worker_TP0 pid=69898) INFO 12-19 16:52:52 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/48c10e1f97/rank_0_0/backbone for vLLM's torch.compile
(Worker_TP0 pid=69898) INFO 12-19 16:52:52 [backends.py:694] Dynamo bytecode transform time: 5.46 s
(Worker_TP1 pid=69899) INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP0 pid=69898) INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP1 pid=69899) WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
(Worker_TP0 pid=69898) WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
(Worker_TP0 pid=69898) INFO 12-19 16:52:57 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.62 s
(Worker_TP0 pid=69898) INFO 12-19 16:52:57 [monitor.py:34] torch.compile takes 8.08 s in total
(Worker_TP0 pid=69898) INFO 12-19 16:53:00 [gpu_worker.py:375] Available KV cache memory: 22.22 GiB
(EngineCore_DP0 pid=69815) INFO 12-19 16:53:00 [kv_cache_utils.py:1291] GPU KV cache size: 485,344 tokens
(EngineCore_DP0 pid=69815) INFO 12-19 16:53:00 [kv_cache_utils.py:1296] Maximum concurrency for 24,576 tokens per request: 19.75x
(Worker_TP0 pid=69898)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:09, 1.97it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:01<00:08, 1.99it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:08, 1.99it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:07, 2.03it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:06, 2.06it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:06, 2.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:03<00:05, 2.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:05, 2.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:04<00:04, 2.02it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:04, 2.00it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:05<00:04, 1.98it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:06<00:03, 1.96it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:06<00:03, 1.94it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:07<00:02, 1.93it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:07<00:02, 1.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:08<00:01, 1.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:08<00:01, 1.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:09<00:00, 1.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.97it/s]
(Worker_TP0 pid=69898)
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:05, 1.75it/s]
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:04, 2.07it/s]
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.18it/s]
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.24it/s]
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.28it/s]
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.33it/s]
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.34it/s]
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.34it/s]
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.33it/s]
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.33it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.34it/s]
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.28it/s]
(Worker_TP0 pid=69898) INFO 12-19 16:53:16 [gpu_model_runner.py:4610] Graph capturing finished in 15 secs, took 0.76 GiB
(EngineCore_DP0 pid=69815) INFO 12-19 16:53:16 [core.py:259] init engine (profile, create kv cache, warmup model) took 29.33 seconds
(APIServer pid=69653) INFO 12-19 16:53:17 [api_server.py:1099] Supported tasks: ['generate']
(APIServer pid=69653) WARNING 12-19 16:53:17 [model.py:1462] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
(APIServer pid=69653) INFO 12-19 16:53:17 [serving_responses.py:201] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
(APIServer pid=69653) INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
(APIServer pid=69653) INFO 12-19 16:53:17 [serving_completion.py:77] Using default completion sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
(APIServer pid=69653) INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
(APIServer pid=69653) INFO 12-19 16:53:17 [api_server.py:1425] Starting vLLM API server 0 on http://127.0.0.1:8000
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:38] Available routes are:
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /docs, Methods: HEAD, GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /tokenize, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /detokenize, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /pause, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /resume, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /is_paused, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /metrics, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /health, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /load, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/models, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /version, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/messages, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/completions, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: GET
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /invocations, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /classify, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/embeddings, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /score, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/score, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /rerank, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/rerank, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /v2/rerank, Methods: POST
(APIServer pid=69653) INFO 12-19 16:53:17 [launcher.py:46] Route: /pooling, Methods: POST
(APIServer pid=69653) INFO: Started server process [69653]
(APIServer pid=69653) INFO: Waiting for application startup.
(APIServer pid=69653) INFO: Application startup complete.
(APIServer pid=69653) INFO: 127.0.0.1:36624 - "GET /v1/models HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:53:38 [loggers.py:248] Engine 000: Avg prompt throughput: 428.2 tokens/s, Avg generation throughput: 153.1 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:53:48 [loggers.py:248] Engine 000: Avg prompt throughput: 1264.2 tokens/s, Avg generation throughput: 435.2 tokens/s, Running: 38 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:53:58 [loggers.py:248] Engine 000: Avg prompt throughput: 925.9 tokens/s, Avg generation throughput: 700.7 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:08 [loggers.py:248] Engine 000: Avg prompt throughput: 643.5 tokens/s, Avg generation throughput: 815.1 tokens/s, Running: 54 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:18 [loggers.py:248] Engine 000: Avg prompt throughput: 662.7 tokens/s, Avg generation throughput: 855.4 tokens/s, Running: 51 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:28 [loggers.py:248] Engine 000: Avg prompt throughput: 978.9 tokens/s, Avg generation throughput: 826.0 tokens/s, Running: 53 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:38 [loggers.py:248] Engine 000: Avg prompt throughput: 936.6 tokens/s, Avg generation throughput: 896.8 tokens/s, Running: 58 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:48 [loggers.py:248] Engine 000: Avg prompt throughput: 480.3 tokens/s, Avg generation throughput: 907.4 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:54:58 [loggers.py:248] Engine 000: Avg prompt throughput: 919.2 tokens/s, Avg generation throughput: 935.0 tokens/s, Running: 63 reqs, Waiting: 5 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:08 [loggers.py:248] Engine 000: Avg prompt throughput: 627.6 tokens/s, Avg generation throughput: 1008.0 tokens/s, Running: 55 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:18 [loggers.py:248] Engine 000: Avg prompt throughput: 1008.7 tokens/s, Avg generation throughput: 869.9 tokens/s, Running: 50 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 0.8%
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:28 [loggers.py:248] Engine 000: Avg prompt throughput: 756.8 tokens/s, Avg generation throughput: 756.5 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 0.8%
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:38 [loggers.py:248] Engine 000: Avg prompt throughput: 690.0 tokens/s, Avg generation throughput: 750.2 tokens/s, Running: 44 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.7%
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:48 [loggers.py:248] Engine 000: Avg prompt throughput: 757.2 tokens/s, Avg generation throughput: 808.7 tokens/s, Running: 46 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 0.9%
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:55:58 [loggers.py:248] Engine 000: Avg prompt throughput: 840.8 tokens/s, Avg generation throughput: 722.0 tokens/s, Running: 35 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 0.8%
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:56:08 [loggers.py:248] Engine 000: Avg prompt throughput: 823.4 tokens/s, Avg generation throughput: 727.0 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.8%
(APIServer pid=69653) INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO 12-19 16:56:18 [loggers.py:248] Engine 000: Avg prompt throughput: 495.1 tokens/s, Avg generation throughput: 806.8 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 0.7%
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60354 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60358 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:60376 - "POST /v1/completions HTTP/1.1" 200 OK
(APIServer pid=69653) INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 498.7602350669986,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 2.004971386433142,
"tokens_per_second": 1486.353457790027
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_server.log
+941
View File
@@ -0,0 +1,941 @@
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
WARNING 12-19 17:11:45 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
(APIServer pid=76251) INFO 12-19 17:11:45 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
(APIServer pid=76251) INFO 12-19 17:11:45 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'trust_remote_code': True, 'max_model_len': 16384, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 32}
(APIServer pid=76251) The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
(APIServer pid=76251) INFO 12-19 17:11:49 [model.py:514] Resolved architecture: Qwen3NextForCausalLM
(APIServer pid=76251) INFO 12-19 17:11:49 [model.py:1636] Using max model len 16384
(APIServer pid=76251) INFO 12-19 17:11:49 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
(APIServer pid=76251) INFO 12-19 17:11:49 [config.py:312] Disabling cascade attention since it is not supported for hybrid models.
(APIServer pid=76251) INFO 12-19 17:11:49 [config.py:439] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size.
(APIServer pid=76251) INFO 12-19 17:11:49 [config.py:463] Padding mamba page size by 1.49% to ensure that mamba page size and attention page size are exactly equal.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
(EngineCore_DP0 pid=76413) INFO 12-19 17:11:53 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
(EngineCore_DP0 pid=76413) WARNING 12-19 17:11:53 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
INFO 12-19 17:11:57 [pynccl.py:111] vLLM is using nccl==2.27.3
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit...
(Worker_TP1 pid=76496) WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
(Worker_TP0 pid=76495) WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
(Worker_TP1 pid=76496) INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP0 pid=76495) INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 0% Completed | 0/10 [00:00<?, ?it/s]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 10% Completed | 1/10 [00:03<00:32, 3.65s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 20% Completed | 2/10 [00:07<00:29, 3.64s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 30% Completed | 3/10 [00:11<00:26, 3.79s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 40% Completed | 4/10 [00:15<00:23, 3.97s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 50% Completed | 5/10 [00:18<00:18, 3.76s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 60% Completed | 6/10 [00:22<00:15, 3.85s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 70% Completed | 7/10 [00:26<00:11, 3.83s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 80% Completed | 8/10 [00:30<00:07, 3.84s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 2.97s/it]
(Worker_TP0 pid=76495)
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 3.46s/it]
(Worker_TP0 pid=76495)
(Worker_TP0 pid=76495) INFO 12-19 17:12:33 [default_loader.py:308] Loading weights took 34.64 seconds
(Worker_TP0 pid=76495) INFO 12-19 17:12:34 [gpu_model_runner.py:3659] Model loading took 23.5020 GiB memory and 35.701153 seconds
(Worker_TP0 pid=76495) INFO 12-19 17:12:38 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/671837fea6/rank_0_0/backbone for vLLM's torch.compile
(Worker_TP0 pid=76495) INFO 12-19 17:12:38 [backends.py:694] Dynamo bytecode transform time: 4.08 s
(Worker_TP1 pid=76496) INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP0 pid=76495) INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
(Worker_TP1 pid=76496) WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
(Worker_TP0 pid=76495) WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
(Worker_TP0 pid=76495) INFO 12-19 17:12:43 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.59 s
(Worker_TP0 pid=76495) INFO 12-19 17:12:43 [monitor.py:34] torch.compile takes 6.67 s in total
(Worker_TP1 pid=76496) WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_1_0/model, error:
(Worker_TP0 pid=76495) WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_0_0/model, error:
(Worker_TP0 pid=76495) INFO 12-19 17:12:44 [gpu_worker.py:375] Available KV cache memory: 7.09 GiB
(EngineCore_DP0 pid=76413) INFO 12-19 17:12:45 [kv_cache_utils.py:1291] GPU KV cache size: 154,496 tokens
(EngineCore_DP0 pid=76413) INFO 12-19 17:12:45 [kv_cache_utils.py:1296] Maximum concurrency for 16,384 tokens per request: 33.47x
(Worker_TP0 pid=76495)
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 9%|▉ | 1/11 [00:00<00:05, 1.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:01<00:04, 1.83it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 3/11 [00:01<00:04, 1.91it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:02<00:03, 1.93it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 5/11 [00:02<00:03, 1.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:03<00:02, 1.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▎ | 7/11 [00:03<00:02, 1.97it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:04<00:01, 1.98it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:04<00:01, 1.95it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:05<00:00, 1.92it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.98it/s]
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.95it/s]
(Worker_TP0 pid=76495)
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
(Worker_TP1 pid=76496) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
(Worker_TP0 pid=76495) ERROR 12-19 17:12:51 [multiproc_executor.py:826]
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] EngineCore failed to start.
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] Traceback (most recent call last):
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] super().__init__(
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] )
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] vllm_config
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] )
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] return aggregate(get_response())
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~^^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] raise RuntimeError(
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ...<2 lines>...
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] )
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] RuntimeError: Worker failed with error 'at 106:17:
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] if USE_ALIBI_SLOPES:
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] alibi_slope = tl.load(
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] )
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866]
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866]
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] # iterate through tiles
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] for j in range(0, num_blocks):
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866]
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] offs_n = tl.arange(0, BLOCK_SIZE)
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] ^
(EngineCore_DP0 pid=76413) ERROR 12-19 17:12:51 [core.py:866] arange's range must be a power of 2', please check the stack trace above for the root cause
(EngineCore_DP0 pid=76413) Process EngineCore_DP0:
(EngineCore_DP0 pid=76413) Traceback (most recent call last):
(EngineCore_DP0 pid=76413) File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
(EngineCore_DP0 pid=76413) self.run()
(EngineCore_DP0 pid=76413) ~~~~~~~~^^
(EngineCore_DP0 pid=76413) File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
(EngineCore_DP0 pid=76413) self._target(*self._args, **self._kwargs)
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
(EngineCore_DP0 pid=76413) raise e
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
(EngineCore_DP0 pid=76413) engine_core = EngineCoreProc(*args, **kwargs)
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
(EngineCore_DP0 pid=76413) super().__init__(
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=76413) vllm_config, executor_class, log_stats, executor_fail_callback
(EngineCore_DP0 pid=76413) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) )
(EngineCore_DP0 pid=76413) ^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
(EngineCore_DP0 pid=76413) num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~~~~~~~~^
(EngineCore_DP0 pid=76413) vllm_config
(EngineCore_DP0 pid=76413) ^^^^^^^^^^^
(EngineCore_DP0 pid=76413) )
(EngineCore_DP0 pid=76413) ^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
(EngineCore_DP0 pid=76413) self.model_executor.initialize_from_config(kv_cache_configs)
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
(EngineCore_DP0 pid=76413) self.collective_rpc("compile_or_warm_up_model")
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
(EngineCore_DP0 pid=76413) return aggregate(get_response())
(EngineCore_DP0 pid=76413) ~~~~~~~~~~~~^^
(EngineCore_DP0 pid=76413) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
(EngineCore_DP0 pid=76413) raise RuntimeError(
(EngineCore_DP0 pid=76413) ...<2 lines>...
(EngineCore_DP0 pid=76413) )
(EngineCore_DP0 pid=76413) RuntimeError: Worker failed with error 'at 106:17:
(EngineCore_DP0 pid=76413) if USE_ALIBI_SLOPES:
(EngineCore_DP0 pid=76413) alibi_slope = tl.load(
(EngineCore_DP0 pid=76413) alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
(EngineCore_DP0 pid=76413) )
(EngineCore_DP0 pid=76413)
(EngineCore_DP0 pid=76413) num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
(EngineCore_DP0 pid=76413)
(EngineCore_DP0 pid=76413) # iterate through tiles
(EngineCore_DP0 pid=76413) for j in range(0, num_blocks):
(EngineCore_DP0 pid=76413) physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
(EngineCore_DP0 pid=76413)
(EngineCore_DP0 pid=76413) offs_n = tl.arange(0, BLOCK_SIZE)
(EngineCore_DP0 pid=76413) ^
(EngineCore_DP0 pid=76413) arange's range must be a power of 2', please check the stack trace above for the root cause
(Worker_TP0 pid=76495) INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
(Worker_TP1 pid=76496) INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
(APIServer pid=76251) Traceback (most recent call last):
(APIServer pid=76251) File "/opt/venv/bin/vllm", line 7, in <module>
(APIServer pid=76251) sys.exit(main())
(APIServer pid=76251) ~~~~^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
(APIServer pid=76251) args.dispatch_function(args)
(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
(APIServer pid=76251) uvloop.run(run_server(args))
(APIServer pid=76251) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
(APIServer pid=76251) return __asyncio.run(
(APIServer pid=76251) ~~~~~~~~~~~~~^
(APIServer pid=76251) wrapper(),
(APIServer pid=76251) ^^^^^^^^^^
(APIServer pid=76251) ...<2 lines>...
(APIServer pid=76251) **run_kwargs
(APIServer pid=76251) ^^^^^^^^^^^^
(APIServer pid=76251) )
(APIServer pid=76251) ^
(APIServer pid=76251) File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
(APIServer pid=76251) return runner.run(main)
(APIServer pid=76251) ~~~~~~~~~~^^^^^^
(APIServer pid=76251) File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
(APIServer pid=76251) return self._loop.run_until_complete(task)
(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
(APIServer pid=76251) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
(APIServer pid=76251) return await main
(APIServer pid=76251) ^^^^^^^^^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
(APIServer pid=76251) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
(APIServer pid=76251) async with build_async_engine_client(
(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=76251) args,
(APIServer pid=76251) ^^^^^
(APIServer pid=76251) client_config=client_config,
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) ) as engine_client:
(APIServer pid=76251) ^
(APIServer pid=76251) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
(APIServer pid=76251) return await anext(self.gen)
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
(APIServer pid=76251) async with build_async_engine_client_from_engine_args(
(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=76251) engine_args,
(APIServer pid=76251) ^^^^^^^^^^^^
(APIServer pid=76251) ...<2 lines>...
(APIServer pid=76251) client_config=client_config,
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) ) as engine:
(APIServer pid=76251) ^
(APIServer pid=76251) File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
(APIServer pid=76251) return await anext(self.gen)
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
(APIServer pid=76251) async_llm = AsyncLLM.from_vllm_config(
(APIServer pid=76251) vllm_config=vllm_config,
(APIServer pid=76251) ...<6 lines>...
(APIServer pid=76251) client_index=client_index,
(APIServer pid=76251) )
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
(APIServer pid=76251) return cls(
(APIServer pid=76251) vllm_config=vllm_config,
(APIServer pid=76251) ...<9 lines>...
(APIServer pid=76251) client_index=client_index,
(APIServer pid=76251) )
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
(APIServer pid=76251) self.engine_core = EngineCoreClient.make_async_mp_client(
(APIServer pid=76251) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
(APIServer pid=76251) vllm_config=vllm_config,
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) ...<4 lines>...
(APIServer pid=76251) client_index=client_index,
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^^^^^^^^^
(APIServer pid=76251) )
(APIServer pid=76251) ^
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client
(APIServer pid=76251) return AsyncMPClient(*client_args)
(APIServer pid=76251) File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__
(APIServer pid=76251) super().__init__(
(APIServer pid=76251) ~~~~~~~~~~~~~~~~^
(APIServer pid=76251) asyncio_mode=True,
(APIServer pid=76251) ^^^^^^^^^^^^^^^^^^
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 14:23:22 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c39b23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af26c427-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38794 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.66 \nPeak output token throughput (tok/s): 398.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 392.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.71 \nMedian TTFT (ms): 61.84 \nP99 TTFT (ms): 180.02 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.85 \nMedian TPOT (ms): 30.79 \nP99 TPOT (ms): 33.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.86 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 55.30 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 14:26:47 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ef7ca647740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-65595a03-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.73 \nTotal input tokens: 145810 \nTotal generated tokens: 152105 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 742.97 \nPeak output token throughput (tok/s): 1252.00 \nPeak concurrent requests: 55.00 \nTotal token throughput (tok/s): 1455.19 \n---------------Time to First Token----------------\nMean TTFT (ms): 81.58 \nMedian TTFT (ms): 70.11 \nP99 TTFT (ms): 184.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.28 \nMedian TPOT (ms): 36.83 \nP99 TPOT (ms): 50.83 \n---------------Inter-token Latency----------------\nMean ITL (ms): 36.98 \nMedian ITL (ms): 34.37 \nP99 ITL (ms): 119.18 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log
+1027
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 389.5727865589979,
"num_requests": 1000,
"total_num_tokens": 736330,
"requests_per_second": 2.5669144111239337,
"tokens_per_second": 1890.096088342886
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:41:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f794120f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-819f7ea7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 189.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 204.80 \nPeak output token throughput (tok/s): 415.00 \nPeak concurrent requests: 10.00 \nTotal token throughput (tok/s): 404.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.83 \nMedian TTFT (ms): 50.95 \nP99 TTFT (ms): 232.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.52 \nMedian TPOT (ms): 19.18 \nP99 TPOT (ms): 24.48 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.42 \nMedian ITL (ms): 18.84 \nP99 ITL (ms): 34.36 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 15:44:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84e0897740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-9c7b74a1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.04 \nTotal input tokens: 145810 \nTotal generated tokens: 152194 \nRequest throughput (req/s): 3.67 \nOutput token throughput (tok/s): 776.36 \nPeak output token throughput (tok/s): 1303.00 \nPeak concurrent requests: 46.00 \nTotal token throughput (tok/s): 1520.15 \n---------------Time to First Token----------------\nMean TTFT (ms): 86.17 \nMedian TTFT (ms): 57.78 \nP99 TTFT (ms): 291.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.21 \nMedian TPOT (ms): 27.26 \nP99 TPOT (ms): 51.74 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.56 \nMedian ITL (ms): 23.38 \nP99 ITL (ms): 140.32 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log
+1034
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 14:42:05 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f57a3c9f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3e31d4e4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 215.02 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.28 \nPeak output token throughput (tok/s): 324.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 362.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 118.15 \nMedian TTFT (ms): 106.93 \nP99 TTFT (ms): 227.27 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.56 \nMedian TPOT (ms): 55.08 \nP99 TPOT (ms): 69.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.82 \nMedian ITL (ms): 54.23 \nP99 ITL (ms): 120.82 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "WARNING 12-19 14:45:51 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff5fdd7740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84255adf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 238.83 \nTotal input tokens: 145540 \nTotal generated tokens: 151955 \nRequest throughput (req/s): 3.01 \nOutput token throughput (tok/s): 636.24 \nPeak output token throughput (tok/s): 960.00 \nPeak concurrent requests: 94.00 \nTotal token throughput (tok/s): 1245.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 1170.97 \nMedian TTFT (ms): 182.43 \nP99 TTFT (ms): 5964.07 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.13 \nMedian TPOT (ms): 74.02 \nP99 TPOT (ms): 87.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 72.97 \nMedian ITL (ms): 68.87 \nP99 ITL (ms): 158.23 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_server.log
+1033
View File
File diff suppressed because one or more lines are too long
benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 604.6067389169984,
"num_requests": 1000,
"total_num_tokens": 738792,
"requests_per_second": 1.6539676712688476,
"tokens_per_second": 1221.9380837920544
}
benchmarks/benchmark_results_amd-r9700-rocm_atten/openai_gpt-oss-20b_tp2_server.log
+437
View File
File diff suppressed because one or more lines are too long
benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f9e1c29aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e4e803cf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.63 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.85 \nPeak output token throughput (tok/s): 350.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 392.04 \n---------------Time to First Token----------------\nMean TTFT (ms): 100.59 \nMedian TTFT (ms): 84.87 \nP99 TTFT (ms): 217.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.46 \nMedian TPOT (ms): 41.30 \nP99 TPOT (ms): 45.56 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.46 \nMedian ITL (ms): 40.41 \nP99 ITL (ms): 81.46 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5f5b1aeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2d3278b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 211.85 \nTotal input tokens: 146694 \nTotal generated tokens: 155558 \nRequest throughput (req/s): 3.40 \nOutput token throughput (tok/s): 734.30 \nPeak output token throughput (tok/s): 1241.00 \nPeak concurrent requests: 68.00 \nTotal Token throughput (tok/s): 1426.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 111.24 \nMedian TTFT (ms): 89.87 \nP99 TTFT (ms): 307.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 52.80 \nMedian TPOT (ms): 52.89 \nP99 TPOT (ms): 76.85 \n---------------Inter-token Latency----------------\nMean ITL (ms): 52.36 \nMedian ITL (ms): 48.02 \nP99 ITL (ms): 172.90 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log
+1042
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-uv+pl/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 499.5056346299998,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 2.00197941859201,
"tokens_per_second": 1484.1354103024892
}
benchmarks/benchmark_results_amd-r9700-uv+pl/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 680.202111824,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 1.4701512721247574,
"tokens_per_second": 1089.8731231693348
}
benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fd4aef3aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fce2ee75-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.29 \nTotal input tokens: 37841 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.79 \nPeak output token throughput (tok/s): 406.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.56 \n---------------Time to First Token----------------\nMean TTFT (ms): 73.39 \nMedian TTFT (ms): 61.33 \nP99 TTFT (ms): 185.43 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.73 \nMedian TPOT (ms): 30.60 \nP99 TPOT (ms): 33.54 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.74 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 50.56 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f082b14aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af5860ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152213 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 744.74 \nPeak output token throughput (tok/s): 1238.00 \nPeak concurrent requests: 53.00 \nTotal Token throughput (tok/s): 1458.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.15 \nMedian TTFT (ms): 63.63 \nP99 TTFT (ms): 179.56 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 35.83 \nMedian TPOT (ms): 35.55 \nP99 TPOT (ms): 48.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.60 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 112.07 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log
+1040
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700-uv+pl/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 354.4166975950002,
"num_requests": 1000,
"total_num_tokens": 736330,
"requests_per_second": 2.821537491844479,
"tokens_per_second": 2077.5827013698454
}
benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc4f7b12ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1dd28fd7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 214.85 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.43 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 19.00 \nTotal Token throughput (tok/s): 362.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 115.51 \nMedian TTFT (ms): 108.55 \nP99 TTFT (ms): 235.09 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 53.14 \nMedian TPOT (ms): 53.86 \nP99 TPOT (ms): 69.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 53.53 \nMedian ITL (ms): 52.81 \nP99 ITL (ms): 120.56 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f58e6eeeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f4f4694-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 236.39 \nTotal input tokens: 145540 \nTotal generated tokens: 152178 \nRequest throughput (req/s): 3.05 \nOutput token throughput (tok/s): 643.77 \nPeak output token throughput (tok/s): 1024.00 \nPeak concurrent requests: 92.00 \nTotal Token throughput (tok/s): 1259.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 850.20 \nMedian TTFT (ms): 141.29 \nP99 TTFT (ms): 5024.92 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.13 \nMedian TPOT (ms): 71.70 \nP99 TPOT (ms): 89.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.90 \nMedian ITL (ms): 66.80 \nP99 ITL (ms): 157.64 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_server.log
+1045
View File
File diff suppressed because one or more lines are too long
benchmarks/benchmark_results_amd-r9700-uv+pl/openai_gpt-oss-20b_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 602.7616506659997,
"num_requests": 1000,
"total_num_tokens": 738792,
"requests_per_second": 1.6590305619063292,
"tokens_per_second": 1225.6785068919007
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84fa08eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84569f25-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 203.97 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.88 \nOutput token throughput (tok/s): 197.56 \nPeak output token throughput (tok/s): 330.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 385.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 108.91 \nMedian TTFT (ms): 95.15 \nP99 TTFT (ms): 230.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.90 \nMedian TPOT (ms): 47.66 \nP99 TPOT (ms): 53.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.94 \nMedian ITL (ms): 46.64 \nP99 ITL (ms): 85.76 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5366d42ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a714d425-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 217.26 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.31 \nOutput token throughput (tok/s): 716.13 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 77.00 \nTotal Token throughput (tok/s): 1391.33 \n---------------Time to First Token----------------\nMean TTFT (ms): 186.47 \nMedian TTFT (ms): 105.59 \nP99 TTFT (ms): 1603.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 59.68 \nMedian TPOT (ms): 59.91 \nP99 TPOT (ms): 84.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 59.35 \nMedian ITL (ms): 54.32 \nP99 ITL (ms): 190.63 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_server.log
+1047
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/RedHatAI_Qwen3-14B-FP8-dynamic_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 544.9470995769998,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 1.8350405035208417,
"tokens_per_second": 1360.3779166371198
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fde305aeca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a3fea59e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.99 \nTotal input tokens: 40429 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 197.07 \nPeak output token throughput (tok/s): 340.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 402.31 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.59 \nMedian TTFT (ms): 82.18 \nP99 TTFT (ms): 271.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 38.86 \nMedian TPOT (ms): 38.92 \nP99 TPOT (ms): 50.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.20 \nMedian ITL (ms): 32.74 \nP99 ITL (ms): 86.49 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f83b1b0aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f36abb16-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 241.29 \nTotal input tokens: 158086 \nTotal generated tokens: 152183 \nRequest throughput (req/s): 2.98 \nOutput token throughput (tok/s): 630.70 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 152.00 \nTotal Token throughput (tok/s): 1285.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 12436.56 \nMedian TTFT (ms): 17115.90 \nP99 TTFT (ms): 24750.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 84.04 \nMedian TPOT (ms): 85.32 \nP99 TPOT (ms): 115.82 \n---------------Inter-token Latency----------------\nMean ITL (ms): 83.72 \nMedian ITL (ms): 75.54 \nP99 ITL (ms): 261.14 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_server.log
+1048
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 878.6614384089999,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 1.1380947840509639,
"tokens_per_second": 859.7532189051877
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f87f9f3eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a1348ed0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.66 \nTotal input tokens: 40429 \nTotal generated tokens: 39127 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 205.22 \nPeak output token throughput (tok/s): 422.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 417.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 89.11 \nMedian TTFT (ms): 64.29 \nP99 TTFT (ms): 255.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.00 \nMedian TPOT (ms): 21.60 \nP99 TPOT (ms): 28.81 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.91 \nMedian ITL (ms): 20.88 \nP99 ITL (ms): 45.88 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8e7772aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-370f24dd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.05 \nTotal input tokens: 158086 \nTotal generated tokens: 153200 \nRequest throughput (req/s): 3.55 \nOutput token throughput (tok/s): 754.51 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 78.00 \nTotal Token throughput (tok/s): 1533.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 222.29 \nMedian TTFT (ms): 103.65 \nP99 TTFT (ms): 2503.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.74 \nMedian TPOT (ms): 55.60 \nP99 TPOT (ms): 83.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.25 \nMedian ITL (ms): 49.31 \nP99 ITL (ms): 204.91 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_server.log
+1074
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-12b-it-FP8-dynamic_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 569.8341109329999,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 1.7548966985543941,
"tokens_per_second": 1325.705122782343
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fab82852ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb2c00d0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.25 \nTotal input tokens: 40429 \nTotal generated tokens: 39398 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 194.80 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 17.00 \nTotal Token throughput (tok/s): 394.70 \n---------------Time to First Token----------------\nMean TTFT (ms): 135.44 \nMedian TTFT (ms): 102.54 \nP99 TTFT (ms): 390.74 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.24 \nMedian TPOT (ms): 46.60 \nP99 TPOT (ms): 56.08 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.00 \nMedian ITL (ms): 45.58 \nP99 ITL (ms): 140.61 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f310eb62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb0dc17a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 320.24 \nTotal input tokens: 158086 \nTotal generated tokens: 152563 \nRequest throughput (req/s): 2.25 \nOutput token throughput (tok/s): 476.40 \nPeak output token throughput (tok/s): 608.00 \nPeak concurrent requests: 304.00 \nTotal Token throughput (tok/s): 970.05 \n---------------Time to First Token----------------\nMean TTFT (ms): 52928.91 \nMedian TTFT (ms): 59992.28 \nP99 TTFT (ms): 104765.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.40 \nMedian TPOT (ms): 60.27 \nP99 TPOT (ms): 90.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 60.58 \nMedian ITL (ms): 53.74 \nP99 ITL (ms): 244.74 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_server.log
+1092
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/RedHatAI_gemma-3-27b-it-FP8-dynamic_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 767.3010607799997,
"num_requests": 1000,
"total_num_tokens": 755432,
"requests_per_second": 1.303269408989804,
"tokens_per_second": 984.5314161719857
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f575fb42ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1e0af39c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40157 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 202.57 \nPeak output token throughput (tok/s): 311.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 396.07 \n---------------Time to First Token----------------\nMean TTFT (ms): 120.83 \nMedian TTFT (ms): 107.92 \nP99 TTFT (ms): 215.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.17 \nMedian TPOT (ms): 44.69 \nP99 TPOT (ms): 56.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.94 \nMedian ITL (ms): 42.97 \nP99 ITL (ms): 110.51 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f193f9faca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5a7df568-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.37 \nTotal input tokens: 146694 \nTotal generated tokens: 153158 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 614.17 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 168.00 \nTotal Token throughput (tok/s): 1202.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 14118.91 \nMedian TTFT (ms): 18607.33 \nP99 TTFT (ms): 31435.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.10 \nMedian TPOT (ms): 90.68 \nP99 TPOT (ms): 104.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.93 \nMedian ITL (ms): 84.69 \nP99 ITL (ms): 190.09 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_server.log
+1054
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 726.8079903329999,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 1.3758792051004176,
"tokens_per_second": 1019.986034633913
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6885e56ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df1894e5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.60 \nTotal input tokens: 38358 \nTotal generated tokens: 39514 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 206.23 \nPeak output token throughput (tok/s): 361.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 406.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.14 \nMedian TTFT (ms): 66.00 \nP99 TTFT (ms): 145.78 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.00 \nMedian TPOT (ms): 28.14 \nP99 TPOT (ms): 35.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.59 \nMedian ITL (ms): 27.66 \nP99 ITL (ms): 54.69 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb8a2696ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e434292b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.64 \nTotal input tokens: 146694 \nTotal generated tokens: 153347 \nRequest throughput (req/s): 3.54 \nOutput token throughput (tok/s): 753.01 \nPeak output token throughput (tok/s): 1160.00 \nPeak concurrent requests: 65.00 \nTotal Token throughput (tok/s): 1473.36 \n---------------Time to First Token----------------\nMean TTFT (ms): 93.13 \nMedian TTFT (ms): 87.23 \nP99 TTFT (ms): 181.31 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.81 \nMedian TPOT (ms): 47.62 \nP99 TPOT (ms): 61.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.43 \nMedian ITL (ms): 44.70 \nP99 ITL (ms): 109.90 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_server.log
+1076
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 449.04138348200104,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 2.2269662369327774,
"tokens_per_second": 1650.9257882903235
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f98e81ceca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b53e2ad8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.26 \nTotal input tokens: 38358 \nTotal generated tokens: 38404 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 195.68 \nPeak output token throughput (tok/s): 304.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 391.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 519.96 \nMedian TTFT (ms): 124.31 \nP99 TTFT (ms): 10120.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 44.04 \nMedian TPOT (ms): 42.30 \nP99 TPOT (ms): 57.88 \n---------------Inter-token Latency----------------\nMean ITL (ms): 43.67 \nMedian ITL (ms): 40.54 \nP99 ITL (ms): 125.33 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4ab18f6ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-72f2dd3c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 351.46 \nTotal input tokens: 146694 \nTotal generated tokens: 148389 \nRequest throughput (req/s): 2.05 \nOutput token throughput (tok/s): 422.20 \nPeak output token throughput (tok/s): 544.00 \nPeak concurrent requests: 369.00 \nTotal Token throughput (tok/s): 839.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 72209.01 \nMedian TTFT (ms): 79268.59 \nP99 TTFT (ms): 146128.91 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.26 \nMedian TPOT (ms): 71.50 \nP99 TPOT (ms): 93.20 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.85 \nMedian ITL (ms): 62.03 \nP99 ITL (ms): 206.26 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_server.log
+1128
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/cpatonn_Qwen3-Next-80B-A3B-Instruct-AWQ-4bit_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 1089.8343978980001,
"num_requests": 1000,
"total_num_tokens": 741334,
"requests_per_second": 0.9175705978162676,
"tokens_per_second": 680.226281561525
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f532a48eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df39931f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.49 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 195.98 \nPeak output token throughput (tok/s): 365.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 386.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.34 \nMedian TTFT (ms): 69.07 \nP99 TTFT (ms): 180.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.10 \nMedian TPOT (ms): 37.08 \nP99 TPOT (ms): 40.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 37.10 \nMedian ITL (ms): 36.27 \nP99 ITL (ms): 58.47 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f081ef62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ba51e591-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 209.75 \nTotal input tokens: 145579 \nTotal generated tokens: 150640 \nRequest throughput (req/s): 3.43 \nOutput token throughput (tok/s): 718.18 \nPeak output token throughput (tok/s): 1259.00 \nPeak concurrent requests: 58.00 \nTotal Token throughput (tok/s): 1412.23 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.92 \nMedian TTFT (ms): 75.54 \nP99 TTFT (ms): 188.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 42.81 \nMedian TPOT (ms): 42.63 \nP99 TPOT (ms): 54.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 42.63 \nMedian ITL (ms): 40.26 \nP99 ITL (ms): 125.13 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_server.log
+1043
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp1_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 410.0767225319996,
"num_requests": 1000,
"total_num_tokens": 736330,
"requests_per_second": 2.438568065569649,
"tokens_per_second": 1795.5908237208996
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps1.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4b1de6aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-073f047f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.90 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 203.07 \nPeak output token throughput (tok/s): 413.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 401.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.57 \nMedian TTFT (ms): 47.42 \nP99 TTFT (ms): 140.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.86 \nMedian TPOT (ms): 21.74 \nP99 TPOT (ms): 23.96 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.82 \nMedian ITL (ms): 21.46 \nP99 ITL (ms): 32.13 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_qps4.0_latency.json
+4
View File
@@ -0,0 +1,4 @@
{
"success": true,
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f791efdeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2ef19409-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.59 \nTotal input tokens: 145810 \nTotal generated tokens: 151842 \nRequest throughput (req/s): 3.66 \nOutput token throughput (tok/s): 772.38 \nPeak output token throughput (tok/s): 1274.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1514.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.73 \nMedian TTFT (ms): 49.35 \nP99 TTFT (ms): 135.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.29 \nMedian TPOT (ms): 24.90 \nP99 TPOT (ms): 34.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.09 \nMedian ITL (ms): 23.53 \nP99 ITL (ms): 78.49 \n==================================================\n"
}
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_server.log
+1065
View File
File diff suppressed because it is too large Load Diff
benchmarks/benchmark_results_amd-r9700/meta-llama_Meta-Llama-3.1-8B-Instruct_tp2_throughput.json
+7
View File
@@ -0,0 +1,7 @@
{
"elapsed_time": 269.04666597899995,
"num_requests": 1000,
"total_num_tokens": 736330,
"requests_per_second": 3.7168273257028708,
"tokens_per_second": 2736.811464734795
}

Some files were not shown because too many files have changed in this diff Show More