x
This commit is contained in:
@@ -0,0 +1,98 @@
|
|||||||
|
name: build-and-publish
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
tag:
|
||||||
|
description: "Image tag to publish (e.g. latest)"
|
||||||
|
required: true
|
||||||
|
default: "latest"
|
||||||
|
|
||||||
|
env:
|
||||||
|
IMAGE_REPO: kyuz0/vllm-therock-gfx1201
|
||||||
|
DOCKER_BUILDKIT: "1"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Put Docker on /mnt
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
echo '{ "data-root": "/mnt/docker" }' | sudo tee /etc/docker/daemon.json
|
||||||
|
sudo systemctl stop docker
|
||||||
|
sudo rm -rf /var/lib/docker || true
|
||||||
|
sudo mkdir -p /mnt/docker
|
||||||
|
sudo systemctl start docker
|
||||||
|
docker info | grep "Docker Root Dir"
|
||||||
|
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Free disk space
|
||||||
|
shell: bash
|
||||||
|
run: |
|
||||||
|
set -euxo pipefail
|
||||||
|
echo "Disk BEFORE:"; df -h
|
||||||
|
sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc || true
|
||||||
|
sudo rm -rf /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go || true
|
||||||
|
docker system prune -af || true
|
||||||
|
docker builder prune -af || true
|
||||||
|
sudo apt-get clean
|
||||||
|
sudo rm -rf /var/lib/apt/lists/*
|
||||||
|
sudo rm -rf /opt/hostedtoolcache
|
||||||
|
echo "Disk AFTER:"; df -h
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v3
|
||||||
|
|
||||||
|
- name: BuildKit GC config (8GB cap)
|
||||||
|
run: |
|
||||||
|
cat > /tmp/buildkitd.toml <<'EOF'
|
||||||
|
[worker.oci]
|
||||||
|
gc = true
|
||||||
|
gckeepstorage = 8000 # MB
|
||||||
|
EOF
|
||||||
|
|
||||||
|
- name: Set up Buildx (with GC)
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
with:
|
||||||
|
buildkitd-flags: --config /tmp/buildkitd.toml
|
||||||
|
|
||||||
|
- name: Log in to Docker Hub
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
|
- name: Docker meta
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: docker.io/${{ env.IMAGE_REPO }}
|
||||||
|
tags: |
|
||||||
|
type=raw,value=${{ github.event.inputs.tag }}
|
||||||
|
type=sha
|
||||||
|
type=raw,value={{date 'YYYYMMDD-HHmmss'}}
|
||||||
|
labels: |
|
||||||
|
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||||
|
org.opencontainers.image.revision=${{ github.sha }}
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
id: build
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
platforms: linux/amd64
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
|
provenance: false
|
||||||
|
sbom: false
|
||||||
|
no-cache: true
|
||||||
|
|
||||||
|
- name: Prune buildx cache
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
docker buildx prune -af --verbose --min-free-space 4gb || true
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
__pycache__
|
||||||
Generated
+5
@@ -0,0 +1,5 @@
|
|||||||
|
# 默认忽略的文件
|
||||||
|
/shelf/
|
||||||
|
/workspace.xml
|
||||||
|
# 基于编辑器的 HTTP 客户端请求
|
||||||
|
/httpRequests/
|
||||||
Generated
+12
@@ -0,0 +1,12 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<module type="PYTHON_MODULE" version="4">
|
||||||
|
<component name="NewModuleRootManager">
|
||||||
|
<content url="file://$MODULE_DIR$" />
|
||||||
|
<orderEntry type="jdk" jdkName="Python 3.13 (base)" jdkType="Python SDK" />
|
||||||
|
<orderEntry type="sourceFolder" forTests="false" />
|
||||||
|
</component>
|
||||||
|
<component name="PyDocumentationSettings">
|
||||||
|
<option name="format" value="PLAIN" />
|
||||||
|
<option name="myDocStringFormat" value="Plain" />
|
||||||
|
</component>
|
||||||
|
</module>
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
<component name="InspectionProjectProfileManager">
|
||||||
|
<profile version="1.0">
|
||||||
|
<option name="myName" value="Project Default" />
|
||||||
|
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||||
|
<option name="ignoredPackages">
|
||||||
|
<list>
|
||||||
|
<option value="python-jose" />
|
||||||
|
<option value="passlib" />
|
||||||
|
<option value="aiosqlite" />
|
||||||
|
</list>
|
||||||
|
</option>
|
||||||
|
</inspection_tool>
|
||||||
|
</profile>
|
||||||
|
</component>
|
||||||
+6
@@ -0,0 +1,6 @@
|
|||||||
|
<component name="InspectionProjectProfileManager">
|
||||||
|
<settings>
|
||||||
|
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||||
|
<version value="1.0" />
|
||||||
|
</settings>
|
||||||
|
</component>
|
||||||
Generated
+7
@@ -0,0 +1,7 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="Black">
|
||||||
|
<option name="sdkName" value="Python 3.13 (base)" />
|
||||||
|
</component>
|
||||||
|
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13 (base)" project-jdk-type="Python SDK" />
|
||||||
|
</project>
|
||||||
Generated
+8
@@ -0,0 +1,8 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="ProjectModuleManager">
|
||||||
|
<modules>
|
||||||
|
<module fileurl="file://$PROJECT_DIR$/.idea/amd-r9700-vllm-toolboxes.iml" filepath="$PROJECT_DIR$/.idea/amd-r9700-vllm-toolboxes.iml" />
|
||||||
|
</modules>
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
Generated
+6
@@ -0,0 +1,6 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project version="4">
|
||||||
|
<component name="VcsDirectoryMappings">
|
||||||
|
<mapping directory="$PROJECT_DIR$" vcs="Git" />
|
||||||
|
</component>
|
||||||
|
</project>
|
||||||
+164
@@ -0,0 +1,164 @@
|
|||||||
|
FROM registry.fedoraproject.org/fedora:43
|
||||||
|
|
||||||
|
# 1. System Base & Build Tools
|
||||||
|
# Added 'gperftools-libs' for tcmalloc (fixes double-free)
|
||||||
|
RUN dnf -y install --setopt=install_weak_deps=False --nodocs \
|
||||||
|
python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \
|
||||||
|
gcc gcc-c++ binutils make ffmpeg-free \
|
||||||
|
cmake ninja-build aria2c tar xz vim nano \
|
||||||
|
libdrm-devel zlib-devel openssl-devel jq \
|
||||||
|
numactl-devel gperftools-libs dialog procps-ng \
|
||||||
|
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||||
|
|
||||||
|
# 2. Install "TheRock" ROCm SDK (Tarball Method)
|
||||||
|
WORKDIR /tmp
|
||||||
|
ARG ROCM_MAJOR_VER=7
|
||||||
|
ARG GFX=gfx120X-all
|
||||||
|
RUN set -euo pipefail; \
|
||||||
|
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
|
||||||
|
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
|
||||||
|
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
|
||||||
|
| tr '<' '\n' \
|
||||||
|
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
|
||||||
|
| sort -V | tail -n1)"; \
|
||||||
|
echo "Downloading Latest Tarball: ${KEY}"; \
|
||||||
|
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \
|
||||||
|
mkdir -p /opt/rocm; \
|
||||||
|
tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \
|
||||||
|
rm therock.tar.gz
|
||||||
|
|
||||||
|
# 3. Configure Global ROCm Environment
|
||||||
|
# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash
|
||||||
|
RUN export ROCM_PATH=/opt/rocm && \
|
||||||
|
BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
|
||||||
|
printf '%s\n' \
|
||||||
|
"export ROCM_PATH=/opt/rocm" \
|
||||||
|
"export HIP_PLATFORM=amd" \
|
||||||
|
"export HIP_PATH=/opt/rocm" \
|
||||||
|
"export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
|
||||||
|
"export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
|
||||||
|
"export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
|
||||||
|
"export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
|
||||||
|
"export ROCBLAS_USE_HIPBLASLT=1" \
|
||||||
|
"export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
|
||||||
|
"export VLLM_TARGET_DEVICE=rocm" \
|
||||||
|
"export HIP_FORCE_DEV_KERNARG=1" \
|
||||||
|
"export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
|
||||||
|
"export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
|
||||||
|
> /etc/profile.d/rocm-sdk.sh && \
|
||||||
|
chmod 0644 /etc/profile.d/rocm-sdk.sh
|
||||||
|
|
||||||
|
# 4. Python Venv Setup
|
||||||
|
RUN /usr/bin/python3.13 -m venv /opt/venv
|
||||||
|
ENV VIRTUAL_ENV=/opt/venv
|
||||||
|
ENV PATH=/opt/venv/bin:$PATH
|
||||||
|
ENV PIP_NO_CACHE_DIR=1
|
||||||
|
RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
|
||||||
|
RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
|
||||||
|
|
||||||
|
# 5. Install PyTorch (TheRock Nightly)
|
||||||
|
RUN python -m pip install \
|
||||||
|
--index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
|
||||||
|
--pre torch torchaudio torchvision
|
||||||
|
|
||||||
|
# Flash-Attention
|
||||||
|
WORKDIR /opt
|
||||||
|
ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
|
||||||
|
|
||||||
|
RUN git clone https://github.com/ROCm/flash-attention.git &&\
|
||||||
|
cd flash-attention &&\
|
||||||
|
git checkout main_perf &&\
|
||||||
|
python setup.py install && \
|
||||||
|
cd /opt && rm -rf /opt/flash-attention
|
||||||
|
|
||||||
|
# 6. Clone vLLM
|
||||||
|
RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
|
||||||
|
WORKDIR /opt/vllm
|
||||||
|
|
||||||
|
# --- PATCHING ---
|
||||||
|
# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers),
|
||||||
|
# vLLM falls back to CPU. We patch it to force ROCm detection.
|
||||||
|
RUN echo "import sys, re" > patch_vllm.py && \
|
||||||
|
echo "from pathlib import Path" >> patch_vllm.py && \
|
||||||
|
# Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks
|
||||||
|
echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
|
||||||
|
echo "txt = p.read_text()" >> patch_vllm.py && \
|
||||||
|
echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
|
||||||
|
echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
|
||||||
|
echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
|
||||||
|
echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
|
||||||
|
echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
|
||||||
|
echo "p.write_text(txt)" >> patch_vllm.py && \
|
||||||
|
# Patch 2: rocm.py - Mock amdsmi and force device name
|
||||||
|
echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
|
||||||
|
echo "txt = p.read_text()" >> patch_vllm.py && \
|
||||||
|
echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
|
||||||
|
echo "txt = header + txt" >> patch_vllm.py && \
|
||||||
|
echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
|
||||||
|
echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
|
||||||
|
echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
|
||||||
|
echo "p.write_text(txt)" >> patch_vllm.py && \
|
||||||
|
echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
|
||||||
|
python patch_vllm.py
|
||||||
|
|
||||||
|
# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
|
||||||
|
RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
|
||||||
|
ENV ROCM_HOME="/opt/rocm"
|
||||||
|
ENV HIP_PATH="/opt/rocm"
|
||||||
|
ENV VLLM_TARGET_DEVICE="rocm"
|
||||||
|
ENV PYTORCH_ROCM_ARCH="gfx1201"
|
||||||
|
ENV HIP_ARCHITECTURES="gfx1201"
|
||||||
|
ENV AMDGPU_TARGETS="gfx1201"
|
||||||
|
ENV MAX_JOBS="4"
|
||||||
|
|
||||||
|
# --- FIX FOR SEGFAULT ---
|
||||||
|
# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC.
|
||||||
|
# This aligns the ABI of the compiled vLLM extensions with PyTorch.
|
||||||
|
ENV CC="/opt/rocm/llvm/bin/clang"
|
||||||
|
ENV CXX="/opt/rocm/llvm/bin/clang++"
|
||||||
|
|
||||||
|
RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
|
||||||
|
echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
|
||||||
|
export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
|
||||||
|
export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
|
||||||
|
python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
|
||||||
|
python -m pip install /tmp/dist/*.whl
|
||||||
|
|
||||||
|
# --- bitsandbytes (ROCm) ---
|
||||||
|
WORKDIR /opt
|
||||||
|
RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
|
||||||
|
WORKDIR /opt/bitsandbytes
|
||||||
|
|
||||||
|
# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
|
||||||
|
ENV HIP_PLATFORM="amd"
|
||||||
|
ENV CMAKE_PREFIX_PATH="/opt/rocm"
|
||||||
|
|
||||||
|
# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
|
||||||
|
RUN cmake -S . \
|
||||||
|
-DGPU_TARGETS="gfx1201" \
|
||||||
|
-DBNB_ROCM_ARCH="gfx1201" \
|
||||||
|
-DCOMPUTE_BACKEND=hip \
|
||||||
|
-DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
|
||||||
|
-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
|
||||||
|
&& \
|
||||||
|
make -j$(nproc) && \
|
||||||
|
python -m pip install --no-cache-dir . --no-build-isolation --no-deps
|
||||||
|
|
||||||
|
# 8. Final Cleanup & Runtime
|
||||||
|
WORKDIR /opt
|
||||||
|
RUN chmod -R a+rwX /opt && \
|
||||||
|
find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
|
||||||
|
find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
|
||||||
|
rm -rf /root/.cache/pip || true && \
|
||||||
|
dnf clean all && rm -rf /var/cache/dnf/*
|
||||||
|
|
||||||
|
COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
|
||||||
|
COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
|
||||||
|
COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
|
||||||
|
COPY scripts/start_vllm.py /usr/local/bin/start-vllm
|
||||||
|
COPY benchmarks/max_context_results.json /opt/max_context_results.json
|
||||||
|
COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
|
||||||
|
RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
|
||||||
|
RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
|
||||||
|
|
||||||
|
CMD ["/bin/bash"]
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
# AMD Radeon 9700 AI PRO (gfx1201) — vLLM Toolbox/Container
|
||||||
|
|
||||||
|
An **fedora-based** Docker/Podman container that is **Toolbx-compatible** (usable as a Fedora toolbox) for serving LLMs with **vLLM** on **AMD Radeon R9700 (gfx1201)**. Built on the TheRock nightly builds for ROCM.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Table of Contents
|
||||||
|
|
||||||
|
* [Tested Models (Benchmarks)](#tested-models-benchmarks)
|
||||||
|
* [1) Toolbx vs Docker/Podman](#1-toolbx-vs-dockerpodman)
|
||||||
|
* [2) Quickstart — Fedora Toolbx](#2-quickstart--fedora-toolbx)
|
||||||
|
* [3) Quickstart — Ubuntu (Distrobox)](#3-quickstart--ubuntu-distrobox)
|
||||||
|
* [4) Testing the API](#4-testing-the-api)
|
||||||
|
* [5) Use a Web UI for Chatting](#5-use-a-web-ui-for-chatting)
|
||||||
|
|
||||||
|
|
||||||
|
## Tested Models (Benchmarks)
|
||||||
|
|
||||||
|
View full benchmarks at: [https://kyuz0.github.io/amd-r9700-vllm-toolboxes/](https://kyuz0.github.io/amd-r9700-vllm-toolboxes/)
|
||||||
|
|
||||||
|
*Run benchmarks now include a comparison between the default Triton backend and the optional ROCm attention backend.*
|
||||||
|
|
||||||
|
|
||||||
|
**Table Key:** Cell values represent `Max Context Length (GPU Memory Utilization)`.
|
||||||
|
|
||||||
|
| Model | TP | 1 Req | 4 Reqs | 8 Reqs | 16 Reqs |
|
||||||
|
| :--- | :--- | :--- | :--- | :--- | :--- |
|
||||||
|
| **`meta-llama/Meta-Llama-3.1-8B-Instruct`** | 1 | 127k (0.98) | 127k (0.98) | 127k (0.98) | 127k (0.98) |
|
||||||
|
| | 2 | 105k (0.98) | 105k (0.98) | 105k (0.98) | 105k (0.98) |
|
||||||
|
| **`openai/gpt-oss-20b`** | 1 | 131k (0.98) | 131k (0.98) | 131k (0.98) | 131k (0.98) |
|
||||||
|
| | 2 | 131k (0.95) | 131k (0.95) | 131k (0.95) | 131k (0.95) |
|
||||||
|
| **`RedHatAI/Qwen3-14B-FP8-dynamic`** | 1 | 41k (0.98) | 41k (0.98) | 41k (0.98) | 41k (0.98) |
|
||||||
|
| | 2 | 41k (0.95) | 41k (0.95) | 41k (0.95) | 41k (0.95) |
|
||||||
|
| **`cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit`** | 1 | 151k (0.98) | 151k (0.98) | 151k (0.98) | 151k (0.98) |
|
||||||
|
| | 2 | 262k (0.98) | 262k (0.98) | 262k (0.98) | 262k (0.98) |
|
||||||
|
| **`cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit`** | 2 | 156k (0.98) | 156k (0.98) | 156k (0.98) | 156k (0.98) |
|
||||||
|
| **`RedHatAI/gemma-3-12b-it-FP8-dynamic`** | 1 | 45k (0.98) | 45k (0.98) | 45k (0.98) | 45k (0.98) |
|
||||||
|
| | 2 | 126k (0.98) | 126k (0.98) | 121k (0.95) | 121k (0.95) |
|
||||||
|
| **`RedHatAI/gemma-3-27b-it-FP8-dynamic`** | 2 | 60k (0.98) | 60k (0.98) | 60k (0.98) | 60k (0.98) |
|
||||||
|
|
||||||
|
### Advanced Tuning
|
||||||
|
|
||||||
|
See [TUNING.md](TUNING.md) for a guide on how to enable undervolting and raise the power limit on AMD R9700 cards on Linux to improve performance and efficiency.
|
||||||
|
|
||||||
|
### 🆕 Update: Comparison of Attention Backends (Triton vs ROCm)
|
||||||
|
|
||||||
|
*Added Support for ROCm Native Attention Backend*
|
||||||
|
|
||||||
|
I have added the ability to switch between the default **Triton** backend and the experimental **ROCm** native backend for attention operations. This provides you with more flexibility to optimize for stability or throughput depending on your specific model and workload.
|
||||||
|
|
||||||
|
| Backend | Stability | Throughput | Compatibility |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **Triton** (Default) | ✅ **High** | 🔸 Good | Works with all tested models |
|
||||||
|
| **ROCm** | ⚠️ **Experimental** | 🚀 **Highest** | May fail with complex architectures |
|
||||||
|
|
||||||
|
**Key Differences:**
|
||||||
|
- **Triton**: The safe choice. It uses the Triton compiler to generate kernels and is the standard for vLLM on AMD.
|
||||||
|
- **ROCm**: Uses composable kernel based attention. In my benchmarks, this often yields higher throughput (tokens/sec) but can be less stable, leading to crashes or "invalid graph" errors on some newer models.
|
||||||
|
|
||||||
|
**How to Use:**
|
||||||
|
1. **Easy Mode**: Select the backend in the `start-vllm` wizard (Item 5 in the menu).
|
||||||
|
2. **Manual Mode**: Export the following environment variables before running `vllm serve`:
|
||||||
|
```bash
|
||||||
|
export VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1
|
||||||
|
export VLLM_USE_TRITON_FLASH_ATTN=0
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1) Toolbx vs Docker/Podman
|
||||||
|
|
||||||
|
The `kyuz0/vllm-therock-gfx1201:latest` image can be used both as:
|
||||||
|
|
||||||
|
* **Fedora Toolbx (recommended for development):** Toolbx shares your **HOME** and user, so models/configs live on the host. Great for iterating quickly while keeping the host clean.
|
||||||
|
* **Docker/Podman (recommended for deployment/perf):** Use for running vLLM as a service (host networking, IPC tuning, etc.). Always **mount a host directory** for model weights so they stay outside the container.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2) Quickstart — Fedora Toolbx
|
||||||
|
|
||||||
|
Create a toolbox that exposes the GPU and relaxes seccomp to avoid ROCm syscall issues:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
toolbox create vllm-r9700 \
|
||||||
|
--image docker.io/kyuz0/vllm-therock-gfx1201:latest \
|
||||||
|
-- --device /dev/dri --device /dev/kfd \
|
||||||
|
--group-add video --group-add render --security-opt seccomp=unconfined
|
||||||
|
```
|
||||||
|
|
||||||
|
Enter it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
toolbox enter vllm-r9700
|
||||||
|
```
|
||||||
|
|
||||||
|
**Model storage:** Models are downloaded to `~/.cache/huggingface` by default. This directory is shared with the host if you created the toolbox correctly, so downloads persist.
|
||||||
|
|
||||||
|
### Serving a Model (Easiest Way)
|
||||||
|
|
||||||
|
The toolbox includes a TUI wizard called **`start-vllm`** which includes pre-configured models and handles launch flags. It also allows you to select the experimental **ROCm attention backend**. This is the easiest way to get started.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# if your weights live on disk instead of on HuggingFace, point the
|
||||||
|
# launcher at the directory that contains the model folders. the
|
||||||
|
# script will look for a subdirectory matching the repo ID and use it
|
||||||
|
# when launching.
|
||||||
|
export LOCAL_MODEL_DIR=/workspace/models
|
||||||
|
start-vllm
|
||||||
|
|
||||||
|
# you can also just run the CLI yourself and pass the path directly:
|
||||||
|
vllm serve /workspace/models/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit ...
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Cache note:** vLLM writes compiled kernels to `~/.cache/vllm/`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### 离线/本地模式
|
||||||
|
|
||||||
|
如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。
|
||||||
|
脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**，
|
||||||
|
不会尝试下载任何内容。适用于没有外网或受限环境的部署。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export LOCAL_MODEL_DIR=/workspace/models
|
||||||
|
start-vllm # 本地模式，如果模型缺失则失败
|
||||||
|
```
|
||||||
|
|
||||||
|
### 离线/本地模式
|
||||||
|
|
||||||
|
如果你希望完全禁用网络访问，只使用本地权重，可以利用 `LOCAL_MODEL_DIR`。
|
||||||
|
脚本会在该目录下查找模型子目录；**找不到时会立即报错并退出**，
|
||||||
|
不会尝试下载任何内容。适用于没有外网或受限环境的部署。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export LOCAL_MODEL_DIR=/workspace/models
|
||||||
|
start-vllm # 本地模式，如果模型缺失则失败
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3) Quickstart — Ubuntu (Distrobox)
|
||||||
|
|
||||||
|
Ubuntu’s toolbox package still breaks GPU access, so use Distrobox instead:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
distrobox create -n vllm-r9700 \
|
||||||
|
--image docker.io/kyuz0/vllm-therock-gfx1201:latest \
|
||||||
|
--additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined"
|
||||||
|
|
||||||
|
distrobox enter vllm-r9700
|
||||||
|
```
|
||||||
|
|
||||||
|
> **Verification:** Run `rocm-smi` to check GPU status.
|
||||||
|
|
||||||
|
### Serving a Model
|
||||||
|
Same as above, you can use the **`start-vllm`** wizard to launch models easily.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
start-vllm
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4) Testing the API
|
||||||
|
|
||||||
|
Once the server is up, hit the OpenAI‑compatible endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -X POST http://localhost:8000/v1/chat/completions \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello! Test the performance."}]}'
|
||||||
|
```
|
||||||
|
|
||||||
|
You should receive a JSON response with a `choices[0].message.content` reply.
|
||||||
|
|
||||||
|
If you don't want to bother specifying the model name, you can run this which will query the currently deployed model:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
MODEL=$(curl -s http://localhost:8000/v1/models | jq -r '.data[0].id') curl -X POST http://localhost:8000/v1/chat/completions \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d "{
|
||||||
|
\"model\": \"$MODEL\",
|
||||||
|
\"messages\":[{\"role\":\"user\",\"content\":\"Hello! Test the performance.\"}]
|
||||||
|
}"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5) Use a Web UI for Chatting
|
||||||
|
|
||||||
|
If vLLM is on a remote server, expose port 8000 via SSH port forwarding:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -L 0.0.0.0:8000:localhost:8000 <vllm-host>
|
||||||
|
```
|
||||||
|
|
||||||
|
Then, you can start HuggingFace ChatUI like this (on your host):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -p 3000:3000 \
|
||||||
|
--add-host=host.docker.internal:host-gateway \
|
||||||
|
-e OPENAI_BASE_URL=http://host.docker.internal:8000/v1 \
|
||||||
|
-e OPENAI_API_KEY=dummy \
|
||||||
|
-v chat-ui-data:/data \
|
||||||
|
ghcr.io/huggingface/chat-ui-db
|
||||||
|
```
|
||||||
|
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
# AMD Radeon 9700 AI PRO (Navi 48) Tuning Guide
|
||||||
|
|
||||||
|
This guide show you how to enable undervolting and raise the power limit on the Radeon PRO R9700 on Fedora 43.
|
||||||
|
|
||||||
|
## 1. Prerequisites (Enable Overclocking)
|
||||||
|
|
||||||
|
The `ppfeaturemask` must be set to unlock voltage control.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo grubby --update-kernel=ALL --args="amdgpu.ppfeaturemask=0xffffffff"
|
||||||
|
```
|
||||||
|
|
||||||
|
**Reboot your system after running this.**
|
||||||
|
|
||||||
|
## 2. Identify Your GPU
|
||||||
|
|
||||||
|
You need the **PCI Bus ID** of the target card (e.g., `07:00.0`).
|
||||||
|
|
||||||
|
Run this command to list your AMD GPUs:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
lspci -nn | grep "VGA" | grep "AMD"
|
||||||
|
```
|
||||||
|
|
||||||
|
*Example output:*
|
||||||
|
`07:00.0 VGA compatible controller...`
|
||||||
|
*(In this example, your ID is `0000:07:00.0`)*
|
||||||
|
|
||||||
|
## 3. Tuning Script
|
||||||
|
|
||||||
|
Save this script as `tune_r9700.sh`.
|
||||||
|
|
||||||
|
**Important:** Edit the `PCI_ID` variable at the top to match the ID you found in Step 2.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# --- CONFIGURATION ---
|
||||||
|
# Replace this with your specific PCI ID from 'lspci'
|
||||||
|
# Format must be 0000:XX:XX.X
|
||||||
|
PCI_ID="0000:07:00.0"
|
||||||
|
# ---------------------
|
||||||
|
|
||||||
|
# 1. Robustly Find the Card Name (e.g., card1) directly from the PCI Bus
|
||||||
|
# We look inside the PCI device's 'drm' folder for a folder starting with 'card' followed only by numbers.
|
||||||
|
if [ ! -d "/sys/bus/pci/devices/$PCI_ID/drm" ]; then
|
||||||
|
echo "Error: PCI Device $PCI_ID not found or has no DRM driver attached."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# This finds 'card1' but ignores 'card1-DP-6'
|
||||||
|
CARD_NAME=$(ls "/sys/bus/pci/devices/$PCI_ID/drm" | grep -E '^card[0-9]+$' | head -n 1)
|
||||||
|
|
||||||
|
if [ -z "$CARD_NAME" ]; then
|
||||||
|
echo "Error: Could not determine card name for $PCI_ID"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Construct the clean path (e.g., /sys/class/drm/card1)
|
||||||
|
CARD_PATH="/sys/class/drm/$CARD_NAME"
|
||||||
|
|
||||||
|
echo "Tuning GPU: $CARD_NAME (at $CARD_PATH)..."
|
||||||
|
|
||||||
|
# 2. Force Manual Performance Level (Required for UV)
|
||||||
|
# We use 'tee' without pipe to ensure exact errors are caught, but your 'echo | sudo tee' method is fine.
|
||||||
|
echo "manual" | sudo tee "$CARD_PATH/device/power_dpm_force_performance_level" > /dev/null
|
||||||
|
if [ $? -ne 0 ]; then echo "Failed to set Manual mode. Check permissions/path."; exit 1; fi
|
||||||
|
|
||||||
|
# 3. Apply -75mV Undervolt
|
||||||
|
echo "vo -75" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null
|
||||||
|
echo "c" | sudo tee "$CARD_PATH/device/pp_od_clk_voltage" > /dev/null
|
||||||
|
echo "Applied Undervolt (-75mV)"
|
||||||
|
|
||||||
|
# 4. Set Power Limit to 315W
|
||||||
|
# Find the hwmon directory strictly inside the device
|
||||||
|
HWMON_DIR=$(find "$CARD_PATH/device/hwmon/" -maxdepth 1 -name "hwmon*" | head -n 1)
|
||||||
|
if [ -n "$HWMON_DIR" ]; then
|
||||||
|
echo "315000000" | sudo tee "$HWMON_DIR/power1_cap" > /dev/null
|
||||||
|
echo "Applied Power Limit (315W)"
|
||||||
|
else
|
||||||
|
echo "Error: Could not find hwmon directory for power limit."
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
### Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
chmod +x tune_r9700.sh
|
||||||
|
sudo ./tune_r9700.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4\. Verification
|
||||||
|
|
||||||
|
Run these commands to confirm settings are active.
|
||||||
|
|
||||||
|
**Check Undervolt:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Look for: OD_VDDGFX_OFFSET: -75mV
|
||||||
|
cat /sys/class/drm/card0/device/pp_od_clk_voltage
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check Power Limit:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Should be ~315 W
|
||||||
|
cat /sys/class/drm/card0/device/hwmon/hwmon*/power1_cap
|
||||||
|
```
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 14:59:00 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc7c8a23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ce1f54c0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.72 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.76 \nPeak output token throughput (tok/s): 344.00 \nPeak concurrent requests: 16.00 \nTotal token throughput (tok/s): 391.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.14 \nMedian TTFT (ms): 87.12 \nP99 TTFT (ms): 220.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.41 \nMedian TPOT (ms): 41.23 \nP99 TPOT (ms): 46.01 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.43 \nMedian ITL (ms): 40.25 \nP99 ITL (ms): 86.38 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:02:31 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f987a537740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-959ca2a7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 668 \nFailed requests: 52 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 180.01 \nTotal input tokens: 137901 \nTotal generated tokens: 128399 \nRequest throughput (req/s): 3.71 \nOutput token throughput (tok/s): 713.31 \nPeak output token throughput (tok/s): 1215.00 \nPeak concurrent requests: 72.00 \nTotal token throughput (tok/s): 1479.40 \n---------------Time to First Token----------------\nMean TTFT (ms): 123.92 \nMedian TTFT (ms): 95.36 \nP99 TTFT (ms): 403.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.89 \nMedian TPOT (ms): 55.38 \nP99 TPOT (ms): 442.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.97 \nMedian ITL (ms): 49.28 \nP99 ITL (ms): 187.29 \n==================================================\n"
|
||||||
|
}
|
||||||
+16815
File diff suppressed because one or more lines are too long
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 518.4320176600013,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 1.9288932124863885,
|
||||||
|
"tokens_per_second": 1429.9541207853842
|
||||||
|
}
|
||||||
+432
@@ -0,0 +1,432 @@
|
|||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
WARNING 12-19 17:09:03 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:03 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:03 [utils.py:253] non-default args: {'model_tag': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'host': '127.0.0.1', 'model': 'RedHatAI/Qwen3-14B-FP8-dynamic', 'trust_remote_code': True, 'max_model_len': 32000, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [model.py:514] Resolved architecture: Qwen3ForCausalLM
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [model.py:1636] Using max model len 32000
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m INFO 12-19 17:09:08 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:12 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='RedHatAI/Qwen3-14B-FP8-dynamic', speculative_config=None, tokenizer='RedHatAI/Qwen3-14B-FP8-dynamic', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=32000, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=RedHatAI/Qwen3-14B-FP8-dynamic, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m WARNING 12-19 17:09:12 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
|
||||||
|
INFO 12-19 17:09:15 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:35477 backend=nccl
|
||||||
|
INFO 12-19 17:09:16 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||||
|
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||||
|
INFO 12-19 17:09:16 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:17 [gpu_model_runner.py:3562] Starting to load model RedHatAI/Qwen3-14B-FP8-dynamic...
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:17 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.57it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.02it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.14s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.19s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.09s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:22 [default_loader.py:308] Loading weights took 4.43 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:23 [gpu_model_runner.py:3659] Model loading took 7.8555 GiB memory and 5.221202 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:28 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/8760cb82b9/rank_0_0/backbone for vLLM's torch.compile
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:28 [backends.py:694] Dynamo bytecode transform time: 5.45 s
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:31 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:36 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 4.77 s
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:36 [monitor.py:34] torch.compile takes 10.22 s in total
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:39 [gpu_worker.py:375] Available KV cache memory: 22.49 GiB
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:39 [kv_cache_utils.py:1291] GPU KV cache size: 294,752 tokens
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m INFO 12-19 17:09:39 [kv_cache_utils.py:1296] Maximum concurrency for 32,000 tokens per request: 9.21x
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:08, 2.07it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:00<00:07, 2.15it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:07, 2.27it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:06, 2.32it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:05, 2.36it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:05, 2.39it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:02<00:04, 2.42it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:04, 2.44it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:03<00:04, 2.47it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:03, 2.48it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:04<00:03, 2.49it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:04<00:02, 2.50it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:05<00:02, 2.50it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:05<00:02, 2.50it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:06<00:01, 2.49it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:06<00:01, 2.49it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:06<00:00, 2.49it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:07<00:00, 2.48it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.51it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:07<00:00, 2.44it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m
|
||||||
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:06, 1.63it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:01<00:04, 1.99it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.20it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.30it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.36it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.40it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.44it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.47it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.50it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.51it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.53it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.39it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:52 [gpu_model_runner.py:4610] Graph capturing finished in 13 secs, took 0.66 GiB
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 76.00 MiB. GPU 1 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.75 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 167.43 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4302, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampler_output = self.sampler(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits=logits, sampling_metadata=dummy_metadata
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 96, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampled, processed_logprobs = self.sample(logits, sampling_metadata)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/sampler.py", line 187, in sample
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] random_sampled, processed_logprobs = self.topk_topp_sampler(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits,
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<2 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] sampling_metadata.top_p,
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 104, in forward_native
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits = self.apply_top_k_top_p(logits, k, p)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/sample/ops/topk_topp_sampler.py", line 258, in apply_top_k_top_p
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] logits_sort, logits_idx = logits.sort(dim=-1, descending=False)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] torch.OutOfMemoryError: HIP out of memory. Tried to allocate 38.00 MiB. GPU 0 has a total capacity of 31.86 GiB of which 0 bytes is free. Of the allocated memory 30.83 GiB is allocated by PyTorch, with 68.00 MiB allocated in private pools (e.g., HIP Graphs), and 169.24 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 538, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] self.model_runner._dummy_sampler_run(hidden_states=last_hidden_states)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4307, in _dummy_sampler_run
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] raise RuntimeError(
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ...<4 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] ) from e
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826] RuntimeError: CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m ERROR 12-19 17:09:52 [multiproc_executor.py:826]
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] EngineCore failed to start.
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] Traceback (most recent call last):
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] super().__init__(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] vllm_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] return aggregate(get_response())
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ~~~~~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] raise RuntimeError(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] ...<2 lines>...
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ERROR 12-19 17:09:52 [core.py:866] RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m Process EngineCore_DP0:
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m Traceback (most recent call last):
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.run()
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m self._target(*self._args, **self._kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m raise e
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m engine_core = EngineCoreProc(*args, **kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m super().__init__(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m vllm_config, executor_class, log_stats, executor_fail_callback
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m vllm_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.model_executor.initialize_from_config(kv_cache_configs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m self.collective_rpc("compile_or_warm_up_model")
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m return aggregate(get_response())
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ~~~~~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m raise RuntimeError(
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=75534)[0;0m RuntimeError: Worker failed with error 'CUDA out of memory occurred when warming up sampler with 64 dummy requests. Please try lowering `max_num_seqs` or `gpu_memory_utilization` when initializing the engine.', please check the stack trace above for the root cause
|
||||||
|
[0;36m(Worker_TP1 pid=75617)[0;0m INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||||
|
[0;36m(Worker_TP0 pid=75616)[0;0m INFO 12-19 17:09:52 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m Traceback (most recent call last):
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/bin/vllm", line 7, in <module>
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m sys.exit(main())
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m args.dispatch_function(args)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m uvloop.run(run_server(args))
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return __asyncio.run(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m wrapper(),
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m **run_kwargs
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m )
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return runner.run(main)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return self._loop.run_until_complete(task)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return await main
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m async with build_async_engine_client(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m args,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m client_config=client_config,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ) as engine_client:
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return await anext(self.gen)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m async with build_async_engine_client_from_engine_args(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m engine_args,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m client_config=client_config,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ) as engine:
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return await anext(self.gen)
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m vllm_config=vllm_config,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ...<6 lines>...
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m client_index=client_index,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m )
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m return cls(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m vllm_config=vllm_config,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ...<9 lines>...
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m client_index=client_index,
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m )
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client(
|
||||||
|
[0;36m(APIServer pid=75372)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 470.41839592500037,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 2.1257672077931065,
|
||||||
|
"tokens_per_second": 1575.9035072220947
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:31:43 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3319c47740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-242c7681-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.10 \nTotal input tokens: 40429 \nTotal generated tokens: 38744 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 191.71 \nPeak output token throughput (tok/s): 354.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 391.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 127.38 \nMedian TTFT (ms): 107.28 \nP99 TTFT (ms): 286.14 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.15 \nMedian TPOT (ms): 45.01 \nP99 TPOT (ms): 56.92 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.71 \nMedian ITL (ms): 42.72 \nP99 ITL (ms): 114.80 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:35:15 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f567795b740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-8a860b01-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 224.44 \nTotal input tokens: 158086 \nTotal generated tokens: 153768 \nRequest throughput (req/s): 3.21 \nOutput token throughput (tok/s): 685.11 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 111.00 \nTotal token throughput (tok/s): 1389.47 \n---------------Time to First Token----------------\nMean TTFT (ms): 3686.81 \nMedian TTFT (ms): 3310.63 \nP99 TTFT (ms): 10615.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 75.05 \nMedian TPOT (ms): 75.32 \nP99 TPOT (ms): 105.16 \n---------------Inter-token Latency----------------\nMean ITL (ms): 74.21 \nMedian ITL (ms): 62.11 \nP99 ITL (ms): 246.46 \n==================================================\n"
|
||||||
|
}
|
||||||
+1031
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 666.9801866829985,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 1.499294911552266,
|
||||||
|
"tokens_per_second": 1132.6153536237514
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 18:04:12 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb4e3457740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b978cded-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.92 \nTotal input tokens: 40429 \nTotal generated tokens: 38727 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 196.66 \nPeak output token throughput (tok/s): 393.00 \nPeak concurrent requests: 17.00 \nTotal token throughput (tok/s): 401.97 \n---------------Time to First Token----------------\nMean TTFT (ms): 232.41 \nMedian TTFT (ms): 152.06 \nP99 TTFT (ms): 582.73 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 39.99 \nMedian TPOT (ms): 38.66 \nP99 TPOT (ms): 75.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.94 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 310.76 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 18:07:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f3b12e0f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3d8d3c55-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 299.13 \nTotal input tokens: 158086 \nTotal generated tokens: 152394 \nRequest throughput (req/s): 2.41 \nOutput token throughput (tok/s): 509.46 \nPeak output token throughput (tok/s): 1088.00 \nPeak concurrent requests: 294.00 \nTotal token throughput (tok/s): 1037.94 \n---------------Time to First Token----------------\nMean TTFT (ms): 37309.58 \nMedian TTFT (ms): 42371.37 \nP99 TTFT (ms): 86881.21 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 113.18 \nMedian TPOT (ms): 113.38 \nP99 TPOT (ms): 240.19 \n---------------Inter-token Latency----------------\nMean ITL (ms): 109.51 \nMedian ITL (ms): 58.62 \nP99 ITL (ms): 472.97 \n==================================================\n"
|
||||||
|
}
|
||||||
+1053
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 581.3710580350016,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 1.72007186491178,
|
||||||
|
"tokens_per_second": 1299.397329054036
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 17:37:17 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f2ac681f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-0a333e52-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 45 \nFailed requests: 135 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 180.00 \nTotal input tokens: 10566 \nTotal generated tokens: 7279 \nRequest throughput (req/s): 0.25 \nOutput token throughput (tok/s): 40.44 \nPeak output token throughput (tok/s): 234.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 99.14 \n---------------Time to First Token----------------\nMean TTFT (ms): 235.33 \nMedian TTFT (ms): 121.50 \nP99 TTFT (ms): 734.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 55.13 \nMedian TPOT (ms): 54.64 \nP99 TPOT (ms): 75.71 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.59 \nMedian ITL (ms): 50.79 \nP99 ITL (ms): 355.12 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": false,
|
||||||
|
"raw_output": "WARNING 12-19 17:40:29 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7fcc91f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-660ac132-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\n"
|
||||||
|
}
|
||||||
+2570
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 1215.8245900149996,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 0.8224870661545537,
|
||||||
|
"tokens_per_second": 621.3330493592667
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:19:35 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4356b0b740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb7743a2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.40 \nTotal input tokens: 38358 \nTotal generated tokens: 40237 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.79 \nPeak output token throughput (tok/s): 315.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 392.20 \n---------------Time to First Token----------------\nMean TTFT (ms): 125.43 \nMedian TTFT (ms): 115.75 \nP99 TTFT (ms): 233.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 51.71 \nMedian TPOT (ms): 52.62 \nP99 TPOT (ms): 60.97 \n---------------Inter-token Latency----------------\nMean ITL (ms): 51.22 \nMedian ITL (ms): 51.73 \nP99 ITL (ms): 115.89 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:23:04 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8a19abf740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-783b8bf2-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.51 \nTotal input tokens: 146694 \nTotal generated tokens: 152753 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 612.20 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 171.00 \nTotal token throughput (tok/s): 1200.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 14305.88 \nMedian TTFT (ms): 18966.25 \nP99 TTFT (ms): 31290.19 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.19 \nMedian TPOT (ms): 90.39 \nP99 TPOT (ms): 108.34 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.87 \nMedian ITL (ms): 84.16 \nP99 ITL (ms): 193.01 \n==================================================\n"
|
||||||
|
}
|
||||||
+1038
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 723.7066626509986,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 1.3817753125788776,
|
||||||
|
"tokens_per_second": 1024.3570195753498
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 16:03:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fbdd8daf740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2904e4f8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 192.44 \nTotal input tokens: 38358 \nTotal generated tokens: 39896 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 207.32 \nPeak output token throughput (tok/s): 372.00 \nPeak concurrent requests: 15.00 \nTotal token throughput (tok/s): 406.64 \n---------------Time to First Token----------------\nMean TTFT (ms): 97.06 \nMedian TTFT (ms): 80.47 \nP99 TTFT (ms): 208.68 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 32.90 \nMedian TPOT (ms): 32.05 \nP99 TPOT (ms): 41.21 \n---------------Inter-token Latency----------------\nMean ITL (ms): 32.45 \nMedian ITL (ms): 30.18 \nP99 ITL (ms): 85.86 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 16:53:26 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f094443f740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-c9db55f5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 206.61 \nTotal input tokens: 146694 \nTotal generated tokens: 153351 \nRequest throughput (req/s): 3.48 \nOutput token throughput (tok/s): 742.22 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 76.00 \nTotal token throughput (tok/s): 1452.22 \n---------------Time to First Token----------------\nMean TTFT (ms): 194.04 \nMedian TTFT (ms): 127.11 \nP99 TTFT (ms): 1506.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 58.93 \nMedian TPOT (ms): 59.18 \nP99 TPOT (ms): 82.75 \n---------------Inter-token Latency----------------\nMean ITL (ms): 58.36 \nMedian ITL (ms): 52.92 \nP99 ITL (ms): 178.94 \n==================================================\n"
|
||||||
|
}
|
||||||
+846
@@ -0,0 +1,846 @@
|
|||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
WARNING 12-19 16:52:23 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:23 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:23 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', 'trust_remote_code': True, 'max_model_len': 24576, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 64}
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [model.py:514] Resolved architecture: Qwen3MoeForCausalLM
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [model.py:1636] Using max model len 24576
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:52:28 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:52:32 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=24576, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit, enable_prefix_caching=True, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 128, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||||
|
[0;36m(EngineCore_DP0 pid=69815)[0;0m WARNING 12-19 16:52:32 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
|
||||||
|
INFO 12-19 16:52:35 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:49789 backend=nccl
|
||||||
|
INFO 12-19 16:52:35 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||||
|
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||||
|
INFO 12-19 16:52:36 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:36 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit...
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [compressed_tensors_wNa16.py:114] Using ConchLinearKernel for CompressedTensorsWNA16
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:37 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m WARNING 12-19 16:52:37 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 1.61it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:03, 1.60s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:05<00:01, 1.90s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 2.13s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:07<00:00, 1.91s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:46 [default_loader.py:308] Loading weights took 7.71 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:46 [gpu_model_runner.py:3659] Model loading took 8.1992 GiB memory and 9.032124 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:52 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/48c10e1f97/rank_0_0/backbone for vLLM's torch.compile
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:52 [backends.py:694] Dynamo bytecode transform time: 5.46 s
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:55 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP1 pid=69899)[0;0m WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m WARNING 12-19 16:52:55 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=128,N=384,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:57 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.62 s
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:52:57 [monitor.py:34] torch.compile takes 8.08 s in total
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:53:00 [gpu_worker.py:375] Available KV cache memory: 22.22 GiB
|
||||||
|
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:00 [kv_cache_utils.py:1291] GPU KV cache size: 485,344 tokens
|
||||||
|
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:00 [kv_cache_utils.py:1296] Maximum concurrency for 24,576 tokens per request: 19.75x
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/19 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 5%|▌ | 1/19 [00:00<00:09, 1.97it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 11%|█ | 2/19 [00:01<00:08, 1.99it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 3/19 [00:01<00:08, 1.99it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 21%|██ | 4/19 [00:01<00:07, 2.03it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 26%|██▋ | 5/19 [00:02<00:06, 2.06it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 32%|███▏ | 6/19 [00:02<00:06, 2.02it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 37%|███▋ | 7/19 [00:03<00:05, 2.02it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 42%|████▏ | 8/19 [00:03<00:05, 2.02it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 47%|████▋ | 9/19 [00:04<00:04, 2.02it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 53%|█████▎ | 10/19 [00:04<00:04, 2.00it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 58%|█████▊ | 11/19 [00:05<00:04, 1.98it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 12/19 [00:06<00:03, 1.96it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 68%|██████▊ | 13/19 [00:06<00:03, 1.94it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 74%|███████▎ | 14/19 [00:07<00:02, 1.93it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 79%|███████▉ | 15/19 [00:07<00:02, 1.91it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 84%|████████▍ | 16/19 [00:08<00:01, 1.91it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 89%|████████▉ | 17/19 [00:08<00:01, 1.91it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 95%|█████████▍| 18/19 [00:09<00:00, 1.91it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.95it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 19/19 [00:09<00:00, 1.97it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m
|
||||||
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/11 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 9%|▉ | 1/11 [00:00<00:05, 1.75it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 18%|█▊ | 2/11 [00:00<00:04, 2.07it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 27%|██▋ | 3/11 [00:01<00:03, 2.18it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 36%|███▋ | 4/11 [00:01<00:03, 2.24it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 45%|████▌ | 5/11 [00:02<00:02, 2.28it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 55%|█████▍ | 6/11 [00:02<00:02, 2.33it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 64%|██████▎ | 7/11 [00:03<00:01, 2.34it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 73%|███████▎ | 8/11 [00:03<00:01, 2.34it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 82%|████████▏ | 9/11 [00:03<00:00, 2.33it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 91%|█████████ | 10/11 [00:04<00:00, 2.33it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.34it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 100%|██████████| 11/11 [00:04<00:00, 2.28it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=69898)[0;0m INFO 12-19 16:53:16 [gpu_model_runner.py:4610] Graph capturing finished in 15 secs, took 0.76 GiB
|
||||||
|
[0;36m(EngineCore_DP0 pid=69815)[0;0m INFO 12-19 16:53:16 [core.py:259] init engine (profile, create kv cache, warmup model) took 29.33 seconds
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [api_server.py:1099] Supported tasks: ['generate']
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m WARNING 12-19 16:53:17 [model.py:1462] Default sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_responses.py:201] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_completion.py:77] Using default completion sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [serving_chat.py:137] Using default chat sampling params from model: {'repetition_penalty': 1.05, 'temperature': 0.7, 'top_k': 20, 'top_p': 0.8}
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [api_server.py:1425] Starting vLLM API server 0 on http://127.0.0.1:8000
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:38] Available routes are:
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /openapi.json, Methods: HEAD, GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /docs, Methods: HEAD, GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /docs/oauth2-redirect, Methods: HEAD, GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /redoc, Methods: HEAD, GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /scale_elastic_ep, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /is_scaling_elastic_ep, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /tokenize, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /detokenize, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /inference/v1/generate, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /pause, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /resume, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /is_paused, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /metrics, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /health, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /load, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/models, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /version, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/responses/{response_id}/cancel, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/messages, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/chat/completions, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/completions, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/transcriptions, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/audio/translations, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: GET
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /ping, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /invocations, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /classify, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/embeddings, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /score, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/score, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /rerank, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v1/rerank, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /v2/rerank, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:17 [launcher.py:46] Route: /pooling, Methods: POST
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: Started server process [69653]
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: Waiting for application startup.
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: Application startup complete.
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:36624 - "GET /v1/models HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:38 [loggers.py:248] Engine 000: Avg prompt throughput: 428.2 tokens/s, Avg generation throughput: 153.1 tokens/s, Running: 13 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.7%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:48 [loggers.py:248] Engine 000: Avg prompt throughput: 1264.2 tokens/s, Avg generation throughput: 435.2 tokens/s, Running: 38 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.4%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:53:58 [loggers.py:248] Engine 000: Avg prompt throughput: 925.9 tokens/s, Avg generation throughput: 700.7 tokens/s, Running: 41 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:08 [loggers.py:248] Engine 000: Avg prompt throughput: 643.5 tokens/s, Avg generation throughput: 815.1 tokens/s, Running: 54 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.0%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:18 [loggers.py:248] Engine 000: Avg prompt throughput: 662.7 tokens/s, Avg generation throughput: 855.4 tokens/s, Running: 51 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.9%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:28 [loggers.py:248] Engine 000: Avg prompt throughput: 978.9 tokens/s, Avg generation throughput: 826.0 tokens/s, Running: 53 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.6%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:38 [loggers.py:248] Engine 000: Avg prompt throughput: 936.6 tokens/s, Avg generation throughput: 896.8 tokens/s, Running: 58 reqs, Waiting: 0 reqs, GPU KV cache usage: 4.4%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:48 [loggers.py:248] Engine 000: Avg prompt throughput: 480.3 tokens/s, Avg generation throughput: 907.4 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44400 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44484 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41406 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:54:58 [loggers.py:248] Engine 000: Avg prompt throughput: 919.2 tokens/s, Avg generation throughput: 935.0 tokens/s, Running: 63 reqs, Waiting: 5 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54884 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:08 [loggers.py:248] Engine 000: Avg prompt throughput: 627.6 tokens/s, Avg generation throughput: 1008.0 tokens/s, Running: 55 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.8%, Prefix cache hit rate: 0.0%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44472 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41498 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44412 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44460 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44428 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56952 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54866 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:18 [loggers.py:248] Engine 000: Avg prompt throughput: 1008.7 tokens/s, Avg generation throughput: 869.9 tokens/s, Running: 50 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.5%, Prefix cache hit rate: 0.8%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33824 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54868 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54944 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54850 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:28 [loggers.py:248] Engine 000: Avg prompt throughput: 756.8 tokens/s, Avg generation throughput: 756.5 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.4%, Prefix cache hit rate: 0.8%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:38 [loggers.py:248] Engine 000: Avg prompt throughput: 690.0 tokens/s, Avg generation throughput: 750.2 tokens/s, Running: 44 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.1%, Prefix cache hit rate: 0.7%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33822 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33858 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41410 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:48 [loggers.py:248] Engine 000: Avg prompt throughput: 757.2 tokens/s, Avg generation throughput: 808.7 tokens/s, Running: 46 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.9%, Prefix cache hit rate: 0.9%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56956 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44442 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54904 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41464 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:55:58 [loggers.py:248] Engine 000: Avg prompt throughput: 840.8 tokens/s, Avg generation throughput: 722.0 tokens/s, Running: 35 reqs, Waiting: 0 reqs, GPU KV cache usage: 2.5%, Prefix cache hit rate: 0.8%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54828 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54920 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34034 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33840 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41418 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41524 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41560 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41510 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41556 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54880 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:56:08 [loggers.py:248] Engine 000: Avg prompt throughput: 823.4 tokens/s, Avg generation throughput: 727.0 tokens/s, Running: 45 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.0%, Prefix cache hit rate: 0.8%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54832 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41486 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54922 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41476 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41572 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54814 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34016 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56926 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34014 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44444 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41458 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44448 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54928 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44424 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34052 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41542 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33826 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO 12-19 16:56:18 [loggers.py:248] Engine 000: Avg prompt throughput: 495.1 tokens/s, Avg generation throughput: 806.8 tokens/s, Running: 48 reqs, Waiting: 0 reqs, GPU KV cache usage: 3.3%, Prefix cache hit rate: 0.7%
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54896 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33874 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44382 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44492 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44396 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41440 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:33852 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41452 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41478 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:54846 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56940 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34022 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41536 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56918 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:44506 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34064 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34056 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:41434 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60332 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:56932 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60340 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60354 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60358 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60362 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:60376 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
|
[0;36m(APIServer pid=69653)[0;0m INFO: 127.0.0.1:34048 - "POST /v1/completions HTTP/1.1" 200 OK
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 498.7602350669986,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 2.004971386433142,
|
||||||
|
"tokens_per_second": 1486.353457790027
|
||||||
|
}
|
||||||
+941
@@ -0,0 +1,941 @@
|
|||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
WARNING 12-19 17:11:45 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:45 [api_server.py:1351] vLLM API server version 0.13.0rc2.dev112+g763963aa7.d20251213
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:45 [utils.py:253] non-default args: {'model_tag': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'host': '127.0.0.1', 'model': 'cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', 'trust_remote_code': True, 'max_model_len': 16384, 'tensor_parallel_size': 2, 'gpu_memory_utilization': 0.98, 'max_num_seqs': 32}
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [model.py:514] Resolved architecture: Qwen3NextForCausalLM
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [model.py:1636] Using max model len 16384
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [scheduler.py:228] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:312] Disabling cascade attention since it is not supported for hybrid models.
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:439] Setting attention block size to 544 tokens to ensure that attention page size is >= mamba page size.
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m INFO 12-19 17:11:49 [config.py:463] Padding mamba page size by 1.49% to ensure that mamba page size and attention page size are exactly equal.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:11:53 [core.py:93] Initializing a V1 LLM engine (v0.13.0rc2.dev112+g763963aa7.d20251213) with config: model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', speculative_config=None, tokenizer='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=16384, download_dir=None, load_format=auto, tensor_parallel_size=2, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=True, quantization=compressed-tensors, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False), seed=0, served_model_name=cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'level': None, 'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'splitting_ops': ['vllm::unified_attention', 'vllm::unified_attention_with_output', 'vllm::unified_mla_attention', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::linear_attention', 'vllm::plamo2_mamba_mixer', 'vllm::gdn_attention_core', 'vllm::kda_attention', 'vllm::sparse_attn_indexer'], 'compile_mm_encoder': False, 'compile_sizes': [], 'compile_ranges_split_points': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'eliminate_noops': True, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False}, 'max_cudagraph_capture_size': 64, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False}, 'local_cache_dir': None}
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m WARNING 12-19 17:11:53 [multiproc_executor.py:884] Reducing Torch parallelism from 24 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed.
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
Skipping import of cpp extensions due to incompatible torch version 2.10.0a0+rocm7.11.0a20251210 for torchao version 0.14.1 Please see https://github.com/pytorch/ao/issues/2919 for more info
|
||||||
|
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=0 local_rank=0 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
|
||||||
|
INFO 12-19 17:11:56 [parallel_state.py:1203] world_size=2 rank=1 local_rank=1 distributed_init_method=tcp://127.0.0.1:50157 backend=nccl
|
||||||
|
INFO 12-19 17:11:57 [pynccl.py:111] vLLM is using nccl==2.27.3
|
||||||
|
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 1 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 1, EP rank 1
|
||||||
|
INFO 12-19 17:11:57 [parallel_state.py:1411] rank 0 in world size 2 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank 0
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [gpu_model_runner.py:3562] Starting to load model cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:11:58 [compressed_tensors.py:742] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [layer.py:372] Enabled separate cuda stream for MoE shared_experts
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [compressed_tensors_moe.py:188] Using CompressedTensorsWNA16MoEMethod
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:11:58 [rocm.py:306] Using Rocm Attention backend on V1 engine.
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 0% Completed | 0/10 [00:00<?, ?it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 10% Completed | 1/10 [00:03<00:32, 3.65s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 20% Completed | 2/10 [00:07<00:29, 3.64s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 30% Completed | 3/10 [00:11<00:26, 3.79s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 40% Completed | 4/10 [00:15<00:23, 3.97s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 50% Completed | 5/10 [00:18<00:18, 3.76s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 60% Completed | 6/10 [00:22<00:15, 3.85s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 70% Completed | 7/10 [00:26<00:11, 3.83s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 80% Completed | 8/10 [00:30<00:07, 3.84s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 2.97s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Loading safetensors checkpoint shards: 100% Completed | 10/10 [00:34<00:00, 3.46s/it]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:33 [default_loader.py:308] Loading weights took 34.64 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:34 [gpu_model_runner.py:3659] Model loading took 23.5020 GiB memory and 35.701153 seconds
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:38 [backends.py:634] Using cache directory: /home/kyuz0/.cache/vllm/torch_compile_cache/671837fea6/rank_0_0/backbone for vLLM's torch.compile
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:38 [backends.py:694] Dynamo bytecode transform time: 4.08 s
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:40 [backends.py:261] Cache the graph of compile range (1, 2048) for later use
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:12:41 [fused_moe.py:888] Using default MoE config. Performance might be sub-optimal! Config file not found at ['/opt/venv/lib/python3.13/site-packages/vllm/model_executor/layers/fused_moe/configs/E=512,N=256,device_name=AMD-gfx1201,dtype=int4_w4a16.json']
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:43 [backends.py:278] Compiling a graph for compile range (1, 2048) takes 2.59 s
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:43 [monitor.py:34] torch.compile takes 6.67 s in total
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_1_0/model, error:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m WARNING 12-19 17:12:43 [decorators.py:528] Cannot save aot compilation to path /home/kyuz0/.cache/vllm/torch_aot_compile/8cfc5b4a85b0195581f74785744c13daeda3dd4bc8885cc84b5001d538a63819/rank_0_0/model, error:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:44 [gpu_worker.py:375] Available KV cache memory: 7.09 GiB
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:12:45 [kv_cache_utils.py:1291] GPU KV cache size: 154,496 tokens
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m INFO 12-19 17:12:45 [kv_cache_utils.py:1296] Maximum concurrency for 16,384 tokens per request: 33.47x
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/11 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 9%|▉ | 1/11 [00:00<00:05, 1.95it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 18%|█▊ | 2/11 [00:01<00:04, 1.83it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 27%|██▋ | 3/11 [00:01<00:04, 1.91it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 36%|███▋ | 4/11 [00:02<00:03, 1.93it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 5/11 [00:02<00:03, 1.95it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 55%|█████▍ | 6/11 [00:03<00:02, 1.95it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 64%|██████▎ | 7/11 [00:03<00:02, 1.97it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 73%|███████▎ | 8/11 [00:04<00:01, 1.98it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 82%|████████▏ | 9/11 [00:04<00:01, 1.95it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 91%|█████████ | 10/11 [00:05<00:00, 1.92it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.98it/s]
|
||||||
|
Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 11/11 [00:05<00:00, 1.95it/s]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m
|
||||||
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||||
|
Capturing CUDA graphs (decode, FULL): 0%| | 0/7 [00:00<?, ?it/s]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] WorkerProc hit an exception.
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 43, in wrapper
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/core.py", line 1638, in arange
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return _semantic.arange(start, end)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/language/semantic.py", line 583, in arange
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise ValueError("arange's range must be a power of 2")
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ValueError: arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] The above exception was the direct cause of the following exception:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] Traceback (most recent call last):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 821, in worker_busy_loop
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output = func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_worker.py", line 459, in compile_or_warm_up_model
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4586, in capture_model
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._capture_cudagraphs(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] compilation_cases=compilation_cases_decode,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] cudagraph_runtime_mode=CUDAGraphMode.FULL,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] uniform_decode=True,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4664, in _capture_cudagraphs
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self._dummy_run(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_tokens,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<6 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] activate_lora=activate_lora,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/utils/_contextlib.py", line 124, in decorate_context
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/worker/gpu_model_runner.py", line 4198, in _dummy_run
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] outputs = self.model(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids=input_ids,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] **model_kwargs,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/cuda_graph.py", line 220, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.runnable(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 1231, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] hidden_states = self.model(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] input_ids, positions, intermediate_tensors, inputs_embeds
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/decorators.py", line 376, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.aot_compiled_fn(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_dynamo/aot_compile.py", line 124, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.fn(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/model_executor/models/qwen3_next.py", line 997, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] def forward(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/compilation/caching.py", line 54, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self.optimized_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.99", line 333, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] submod_7 = self.submod_7(getitem_21, s72, getitem_22, getitem_23, getitem_24); getitem_21 = getitem_22 = getitem_23 = submod_7 = None
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 936, in call_wrapped
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._wrapped_call(self, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 455, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] raise e
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/fx/graph_module.py", line 442, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return super(self.cls, obj).__call__(*args, **kwargs) # type: ignore[misc]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1776, in _wrapped_call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._call_impl(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/nn/modules/module.py", line 1787, in _call_impl
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return forward_call(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "<eval_with_key>.107", line 5, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] unified_attention_with_output = torch.ops.vllm.unified_attention_with_output(query_8, key_8, value_9, output_5, 'model.layers.3.self_attn.attn'); query_8 = key_8 = value_9 = output_5 = unified_attention_with_output = None
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/torch/_ops.py", line 1209, in __call__
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return self._op(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/utils/kv_transfer_utils.py", line 39, in wrapper
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return func(*args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/layer.py", line 923, in unified_attention_with_output
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self.impl.forward(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] self,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<7 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_block_scale=output_block_scale,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/attention/backends/rocm_attn.py", line 337, in forward
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] chunked_prefill_paged_decode(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] query=query[:num_actual_tokens],
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<17 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] sinks=self.sinks,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/vllm/attention/ops/chunked_prefill_paged_decode.py", line 356, in chunked_prefill_paged_decode
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel_paged_attention_2d[
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<3 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ](
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] output_ptr=output,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ...<37 lines>...
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] USE_FP8=output_scale is not None,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 419, in <lambda>
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 733, in run
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self._do_compile(key, signature, device, constexprs, options, attrs, warmup)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/runtime/jit.py", line 861, in _do_compile
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] kernel = self.compile(src, target=target, options=options.__dict__)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 300, in compile
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module = src.make_ir(target, options, codegen_fns, module_map, context)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] File "/opt/venv/lib64/python3.13/site-packages/triton/compiler/compiler.py", line 80, in make_ir
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] return ast_to_ttir(self.fn, self, context=context, options=options, codegen_fns=codegen_fns,
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] module_map=module_map)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] triton.compiler.errors.CompilationError: at 106:17:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slope = tl.load(
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] )
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] # iterate through tiles
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] for j in range(0, num_blocks):
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] ^
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826] arange's range must be a power of 2
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m ERROR 12-19 17:12:51 [multiproc_executor.py:826]
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] EngineCore failed to start.
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] Traceback (most recent call last):
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] engine_core = EngineCoreProc(*args, **kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] super().__init__(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] vllm_config, executor_class, log_stats, executor_fail_callback
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] vllm_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] self.model_executor.initialize_from_config(kv_cache_configs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] self.collective_rpc("compile_or_warm_up_model")
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] return aggregate(get_response())
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ~~~~~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] raise RuntimeError(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ...<2 lines>...
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] RuntimeError: Worker failed with error 'at 106:17:
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] alibi_slope = tl.load(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] # iterate through tiles
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] for j in range(0, num_blocks):
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866]
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ERROR 12-19 17:12:51 [core.py:866] arange's range must be a power of 2', please check the stack trace above for the root cause
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m Process EngineCore_DP0:
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m Traceback (most recent call last):
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 313, in _bootstrap
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.run()
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/usr/lib64/python3.13/multiprocessing/process.py", line 108, in run
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m self._target(*self._args, **self._kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 870, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m raise e
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 857, in run_engine_core
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m engine_core = EngineCoreProc(*args, **kwargs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 637, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m super().__init__(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m vllm_config, executor_class, log_stats, executor_fail_callback
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 109, in __init__
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m vllm_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core.py", line 256, in _initialize_kv_caches
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.model_executor.initialize_from_config(kv_cache_configs)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/abstract.py", line 116, in initialize_from_config
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m self.collective_rpc("compile_or_warm_up_model")
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 361, in collective_rpc
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m return aggregate(get_response())
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ~~~~~~~~~~~~^^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/executor/multiproc_executor.py", line 344, in get_response
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m raise RuntimeError(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m RuntimeError: Worker failed with error 'at 106:17:
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m if USE_ALIBI_SLOPES:
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m alibi_slope = tl.load(
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m alibi_slopes_ptr + query_head_idx, mask=head_mask, other=0.0
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m )
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m num_blocks = cdiv_fn(seq_len, BLOCK_SIZE)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m # iterate through tiles
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m for j in range(0, num_blocks):
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m physical_block_idx = tl.load(block_tables_ptr + block_table_offset + j)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m offs_n = tl.arange(0, BLOCK_SIZE)
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m ^
|
||||||
|
[0;36m(EngineCore_DP0 pid=76413)[0;0m arange's range must be a power of 2', please check the stack trace above for the root cause
|
||||||
|
[0;36m(Worker_TP0 pid=76495)[0;0m INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||||
|
[0;36m(Worker_TP1 pid=76496)[0;0m INFO 12-19 17:12:51 [multiproc_executor.py:711] Parent process exited, terminating worker
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m Traceback (most recent call last):
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/bin/vllm", line 7, in <module>
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m sys.exit(main())
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/main.py", line 73, in main
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m args.dispatch_function(args)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/cli/serve.py", line 60, in cmd
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m uvloop.run(run_server(args))
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 96, in run
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return __asyncio.run(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m wrapper(),
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m **run_kwargs
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m )
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 195, in run
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return runner.run(main)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/asyncio/runners.py", line 118, in run
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return self._loop.run_until_complete(task)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/uvloop/__init__.py", line 48, in wrapper
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return await main
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1398, in run_server
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 1417, in run_server_worker
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m async with build_async_engine_client(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m args,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m client_config=client_config,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ) as engine_client:
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return await anext(self.gen)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 172, in build_async_engine_client
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m async with build_async_engine_client_from_engine_args(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m engine_args,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ...<2 lines>...
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m client_config=client_config,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ) as engine:
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/usr/lib64/python3.13/contextlib.py", line 214, in __aenter__
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return await anext(self.gen)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/entrypoints/openai/api_server.py", line 213, in build_async_engine_client_from_engine_args
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m async_llm = AsyncLLM.from_vllm_config(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ...<6 lines>...
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m )
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 215, in from_vllm_config
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return cls(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ...<9 lines>...
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m )
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/async_llm.py", line 134, in __init__
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m self.engine_core = EngineCoreClient.make_async_mp_client(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m vllm_config=vllm_config,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ...<4 lines>...
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m client_index=client_index,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m )
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 121, in make_async_mp_client
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m return AsyncMPClient(*client_args)
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m File "/opt/venv/lib64/python3.13/site-packages/vllm/v1/engine/core_client.py", line 820, in __init__
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m super().__init__(
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ~~~~~~~~~~~~~~~~^
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m asyncio_mode=True,
|
||||||
|
[0;36m(APIServer pid=76251)[0;0m ^^^^^^^^^^^^^^^^^^
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 14:23:22 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f7c39b23740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af26c427-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38794 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.66 \nPeak output token throughput (tok/s): 398.00 \nPeak concurrent requests: 14.00 \nTotal token throughput (tok/s): 392.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.71 \nMedian TTFT (ms): 61.84 \nP99 TTFT (ms): 180.02 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.85 \nMedian TPOT (ms): 30.79 \nP99 TPOT (ms): 33.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.86 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 55.30 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 14:26:47 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7ef7ca647740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-65595a03-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.73 \nTotal input tokens: 145810 \nTotal generated tokens: 152105 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 742.97 \nPeak output token throughput (tok/s): 1252.00 \nPeak concurrent requests: 55.00 \nTotal token throughput (tok/s): 1455.19 \n---------------Time to First Token----------------\nMean TTFT (ms): 81.58 \nMedian TTFT (ms): 70.11 \nP99 TTFT (ms): 184.97 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.28 \nMedian TPOT (ms): 36.83 \nP99 TPOT (ms): 50.83 \n---------------Inter-token Latency----------------\nMean ITL (ms): 36.98 \nMedian ITL (ms): 34.37 \nP99 ITL (ms): 119.18 \n==================================================\n"
|
||||||
|
}
|
||||||
+1027
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 389.5727865589979,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 736330,
|
||||||
|
"requests_per_second": 2.5669144111239337,
|
||||||
|
"tokens_per_second": 1890.096088342886
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:41:39 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f794120f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-819f7ea7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 189.28 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.95 \nOutput token throughput (tok/s): 204.80 \nPeak output token throughput (tok/s): 415.00 \nPeak concurrent requests: 10.00 \nTotal token throughput (tok/s): 404.72 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.83 \nMedian TTFT (ms): 50.95 \nP99 TTFT (ms): 232.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 19.52 \nMedian TPOT (ms): 19.18 \nP99 TPOT (ms): 24.48 \n---------------Inter-token Latency----------------\nMean ITL (ms): 19.42 \nMedian ITL (ms): 18.84 \nP99 ITL (ms): 34.36 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 15:44:56 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84e0897740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-9c7b74a1-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.04 \nTotal input tokens: 145810 \nTotal generated tokens: 152194 \nRequest throughput (req/s): 3.67 \nOutput token throughput (tok/s): 776.36 \nPeak output token throughput (tok/s): 1303.00 \nPeak concurrent requests: 46.00 \nTotal token throughput (tok/s): 1520.15 \n---------------Time to First Token----------------\nMean TTFT (ms): 86.17 \nMedian TTFT (ms): 57.78 \nP99 TTFT (ms): 291.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.21 \nMedian TPOT (ms): 27.26 \nP99 TPOT (ms): 51.74 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.56 \nMedian ITL (ms): 23.38 \nP99 ITL (ms): 140.32 \n==================================================\n"
|
||||||
|
}
|
||||||
+1034
File diff suppressed because it is too large
Load Diff
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 14:42:05 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f57a3c9f740>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-3e31d4e4-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 215.02 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.28 \nPeak output token throughput (tok/s): 324.00 \nPeak concurrent requests: 19.00 \nTotal token throughput (tok/s): 362.52 \n---------------Time to First Token----------------\nMean TTFT (ms): 118.15 \nMedian TTFT (ms): 106.93 \nP99 TTFT (ms): 227.27 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.56 \nMedian TPOT (ms): 55.08 \nP99 TPOT (ms): 69.46 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.82 \nMedian ITL (ms): 54.23 \nP99 ITL (ms): 120.82 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "WARNING 12-19 14:45:51 [attention.py:82] Using VLLM_V1_USE_PREFILL_DECODE_ATTENTION environment variable is deprecated and will be removed in v0.14.0 or v1.0.0, whichever is soonest. Please use --attention-config.use_prefill_decode_attention command line argument or AttentionConfig(use_prefill_decode_attention=...) config field instead.\nNamespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7eff5fdd7740>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84255adf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 238.83 \nTotal input tokens: 145540 \nTotal generated tokens: 151955 \nRequest throughput (req/s): 3.01 \nOutput token throughput (tok/s): 636.24 \nPeak output token throughput (tok/s): 960.00 \nPeak concurrent requests: 94.00 \nTotal token throughput (tok/s): 1245.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 1170.97 \nMedian TTFT (ms): 182.43 \nP99 TTFT (ms): 5964.07 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 73.13 \nMedian TPOT (ms): 74.02 \nP99 TPOT (ms): 87.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 72.97 \nMedian ITL (ms): 68.87 \nP99 ITL (ms): 158.23 \n==================================================\n"
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 604.6067389169984,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 738792,
|
||||||
|
"requests_per_second": 1.6539676712688476,
|
||||||
|
"tokens_per_second": 1221.9380837920544
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f9e1c29aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e4e803cf-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 200.63 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.90 \nOutput token throughput (tok/s): 200.85 \nPeak output token throughput (tok/s): 350.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 392.04 \n---------------Time to First Token----------------\nMean TTFT (ms): 100.59 \nMedian TTFT (ms): 84.87 \nP99 TTFT (ms): 217.06 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 41.46 \nMedian TPOT (ms): 41.30 \nP99 TPOT (ms): 45.56 \n---------------Inter-token Latency----------------\nMean ITL (ms): 41.46 \nMedian ITL (ms): 40.41 \nP99 ITL (ms): 81.46 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5f5b1aeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-d2d3278b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 211.85 \nTotal input tokens: 146694 \nTotal generated tokens: 155558 \nRequest throughput (req/s): 3.40 \nOutput token throughput (tok/s): 734.30 \nPeak output token throughput (tok/s): 1241.00 \nPeak concurrent requests: 68.00 \nTotal Token throughput (tok/s): 1426.75 \n---------------Time to First Token----------------\nMean TTFT (ms): 111.24 \nMedian TTFT (ms): 89.87 \nP99 TTFT (ms): 307.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 52.80 \nMedian TPOT (ms): 52.89 \nP99 TPOT (ms): 76.85 \n---------------Inter-token Latency----------------\nMean ITL (ms): 52.36 \nMedian ITL (ms): 48.02 \nP99 ITL (ms): 172.90 \n==================================================\n"
|
||||||
|
}
|
||||||
+1042
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 499.5056346299998,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 2.00197941859201,
|
||||||
|
"tokens_per_second": 1484.1354103024892
|
||||||
|
}
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 680.202111824,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 1.4701512721247574,
|
||||||
|
"tokens_per_second": 1089.8731231693348
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fd4aef3aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-fce2ee75-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 195.29 \nTotal input tokens: 37841 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 198.79 \nPeak output token throughput (tok/s): 406.00 \nPeak concurrent requests: 13.00 \nTotal Token throughput (tok/s): 392.56 \n---------------Time to First Token----------------\nMean TTFT (ms): 73.39 \nMedian TTFT (ms): 61.33 \nP99 TTFT (ms): 185.43 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 30.73 \nMedian TPOT (ms): 30.60 \nP99 TPOT (ms): 33.54 \n---------------Inter-token Latency----------------\nMean ITL (ms): 30.74 \nMedian ITL (ms): 29.96 \nP99 ITL (ms): 50.56 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f082b14aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-af5860ff-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 204.38 \nTotal input tokens: 145810 \nTotal generated tokens: 152213 \nRequest throughput (req/s): 3.52 \nOutput token throughput (tok/s): 744.74 \nPeak output token throughput (tok/s): 1238.00 \nPeak concurrent requests: 53.00 \nTotal Token throughput (tok/s): 1458.16 \n---------------Time to First Token----------------\nMean TTFT (ms): 75.15 \nMedian TTFT (ms): 63.63 \nP99 TTFT (ms): 179.56 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 35.83 \nMedian TPOT (ms): 35.55 \nP99 TPOT (ms): 48.38 \n---------------Inter-token Latency----------------\nMean ITL (ms): 35.60 \nMedian ITL (ms): 33.55 \nP99 ITL (ms): 112.07 \n==================================================\n"
|
||||||
|
}
|
||||||
+1040
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 354.4166975950002,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 736330,
|
||||||
|
"requests_per_second": 2.821537491844479,
|
||||||
|
"tokens_per_second": 2077.5827013698454
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fc4f7b12ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1dd28fd7-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 214.85 \nTotal input tokens: 38756 \nTotal generated tokens: 39194 \nRequest throughput (req/s): 0.84 \nOutput token throughput (tok/s): 182.43 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 19.00 \nTotal Token throughput (tok/s): 362.82 \n---------------Time to First Token----------------\nMean TTFT (ms): 115.51 \nMedian TTFT (ms): 108.55 \nP99 TTFT (ms): 235.09 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 53.14 \nMedian TPOT (ms): 53.86 \nP99 TPOT (ms): 69.03 \n---------------Inter-token Latency----------------\nMean ITL (ms): 53.53 \nMedian ITL (ms): 52.81 \nP99 ITL (ms): 120.56 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f58e6eeeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='openai/gpt-oss-20b', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2f4f4694-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 236.39 \nTotal input tokens: 145540 \nTotal generated tokens: 152178 \nRequest throughput (req/s): 3.05 \nOutput token throughput (tok/s): 643.77 \nPeak output token throughput (tok/s): 1024.00 \nPeak concurrent requests: 92.00 \nTotal Token throughput (tok/s): 1259.45 \n---------------Time to First Token----------------\nMean TTFT (ms): 850.20 \nMedian TTFT (ms): 141.29 \nP99 TTFT (ms): 5024.92 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.13 \nMedian TPOT (ms): 71.70 \nP99 TPOT (ms): 89.98 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.90 \nMedian ITL (ms): 66.80 \nP99 ITL (ms): 157.64 \n==================================================\n"
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 602.7616506659997,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 738792,
|
||||||
|
"requests_per_second": 1.6590305619063292,
|
||||||
|
"tokens_per_second": 1225.6785068919007
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f84fa08eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-84569f25-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 203.97 \nTotal input tokens: 38358 \nTotal generated tokens: 40296 \nRequest throughput (req/s): 0.88 \nOutput token throughput (tok/s): 197.56 \nPeak output token throughput (tok/s): 330.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 385.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 108.91 \nMedian TTFT (ms): 95.15 \nP99 TTFT (ms): 230.11 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.90 \nMedian TPOT (ms): 47.66 \nP99 TPOT (ms): 53.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.94 \nMedian ITL (ms): 46.64 \nP99 ITL (ms): 85.76 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f5366d42ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/Qwen3-14B-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a714d425-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 217.26 \nTotal input tokens: 146694 \nTotal generated tokens: 155585 \nRequest throughput (req/s): 3.31 \nOutput token throughput (tok/s): 716.13 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 77.00 \nTotal Token throughput (tok/s): 1391.33 \n---------------Time to First Token----------------\nMean TTFT (ms): 186.47 \nMedian TTFT (ms): 105.59 \nP99 TTFT (ms): 1603.40 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 59.68 \nMedian TPOT (ms): 59.91 \nP99 TPOT (ms): 84.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 59.35 \nMedian ITL (ms): 54.32 \nP99 ITL (ms): 190.63 \n==================================================\n"
|
||||||
|
}
|
||||||
+1047
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 544.9470995769998,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 1.8350405035208417,
|
||||||
|
"tokens_per_second": 1360.3779166371198
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fde305aeca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a3fea59e-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.99 \nTotal input tokens: 40429 \nTotal generated tokens: 38821 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 197.07 \nPeak output token throughput (tok/s): 340.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 402.31 \n---------------Time to First Token----------------\nMean TTFT (ms): 104.59 \nMedian TTFT (ms): 82.18 \nP99 TTFT (ms): 271.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 38.86 \nMedian TPOT (ms): 38.92 \nP99 TPOT (ms): 50.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 38.20 \nMedian ITL (ms): 32.74 \nP99 ITL (ms): 86.49 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f83b1b0aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-f36abb16-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 241.29 \nTotal input tokens: 158086 \nTotal generated tokens: 152183 \nRequest throughput (req/s): 2.98 \nOutput token throughput (tok/s): 630.70 \nPeak output token throughput (tok/s): 896.00 \nPeak concurrent requests: 152.00 \nTotal Token throughput (tok/s): 1285.86 \n---------------Time to First Token----------------\nMean TTFT (ms): 12436.56 \nMedian TTFT (ms): 17115.90 \nP99 TTFT (ms): 24750.51 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 84.04 \nMedian TPOT (ms): 85.32 \nP99 TPOT (ms): 115.82 \n---------------Inter-token Latency----------------\nMean ITL (ms): 83.72 \nMedian ITL (ms): 75.54 \nP99 ITL (ms): 261.14 \n==================================================\n"
|
||||||
|
}
|
||||||
+1048
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 878.6614384089999,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 1.1380947840509639,
|
||||||
|
"tokens_per_second": 859.7532189051877
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f87f9f3eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-a1348ed0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.66 \nTotal input tokens: 40429 \nTotal generated tokens: 39127 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 205.22 \nPeak output token throughput (tok/s): 422.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 417.26 \n---------------Time to First Token----------------\nMean TTFT (ms): 89.11 \nMedian TTFT (ms): 64.29 \nP99 TTFT (ms): 255.75 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 22.00 \nMedian TPOT (ms): 21.60 \nP99 TPOT (ms): 28.81 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.91 \nMedian ITL (ms): 20.88 \nP99 ITL (ms): 45.88 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f8e7772aca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-12b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-370f24dd-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.05 \nTotal input tokens: 158086 \nTotal generated tokens: 153200 \nRequest throughput (req/s): 3.55 \nOutput token throughput (tok/s): 754.51 \nPeak output token throughput (tok/s): 1152.00 \nPeak concurrent requests: 78.00 \nTotal Token throughput (tok/s): 1533.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 222.29 \nMedian TTFT (ms): 103.65 \nP99 TTFT (ms): 2503.58 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 54.74 \nMedian TPOT (ms): 55.60 \nP99 TPOT (ms): 83.43 \n---------------Inter-token Latency----------------\nMean ITL (ms): 54.25 \nMedian ITL (ms): 49.31 \nP99 ITL (ms): 204.91 \n==================================================\n"
|
||||||
|
}
|
||||||
+1074
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 569.8341109329999,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 1.7548966985543941,
|
||||||
|
"tokens_per_second": 1325.705122782343
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fab82852ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb2c00d0-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 202.25 \nTotal input tokens: 40429 \nTotal generated tokens: 39398 \nRequest throughput (req/s): 0.89 \nOutput token throughput (tok/s): 194.80 \nPeak output token throughput (tok/s): 308.00 \nPeak concurrent requests: 17.00 \nTotal Token throughput (tok/s): 394.70 \n---------------Time to First Token----------------\nMean TTFT (ms): 135.44 \nMedian TTFT (ms): 102.54 \nP99 TTFT (ms): 390.74 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.24 \nMedian TPOT (ms): 46.60 \nP99 TPOT (ms): 56.08 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.00 \nMedian ITL (ms): 45.58 \nP99 ITL (ms): 140.61 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f310eb62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='RedHatAI/gemma-3-27b-it-FP8-dynamic', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-eb0dc17a-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 320.24 \nTotal input tokens: 158086 \nTotal generated tokens: 152563 \nRequest throughput (req/s): 2.25 \nOutput token throughput (tok/s): 476.40 \nPeak output token throughput (tok/s): 608.00 \nPeak concurrent requests: 304.00 \nTotal Token throughput (tok/s): 970.05 \n---------------Time to First Token----------------\nMean TTFT (ms): 52928.91 \nMedian TTFT (ms): 59992.28 \nP99 TTFT (ms): 104765.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 61.40 \nMedian TPOT (ms): 60.27 \nP99 TPOT (ms): 90.17 \n---------------Inter-token Latency----------------\nMean ITL (ms): 60.58 \nMedian ITL (ms): 53.74 \nP99 ITL (ms): 244.74 \n==================================================\n"
|
||||||
|
}
|
||||||
+1092
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 767.3010607799997,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 755432,
|
||||||
|
"requests_per_second": 1.303269408989804,
|
||||||
|
"tokens_per_second": 984.5314161719857
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f575fb42ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-1e0af39c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.24 \nTotal input tokens: 38358 \nTotal generated tokens: 40157 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 202.57 \nPeak output token throughput (tok/s): 311.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 396.07 \n---------------Time to First Token----------------\nMean TTFT (ms): 120.83 \nMedian TTFT (ms): 107.92 \nP99 TTFT (ms): 215.44 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 45.17 \nMedian TPOT (ms): 44.69 \nP99 TPOT (ms): 56.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 44.94 \nMedian ITL (ms): 42.97 \nP99 ITL (ms): 110.51 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f193f9faca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-5a7df568-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 249.37 \nTotal input tokens: 146694 \nTotal generated tokens: 153158 \nRequest throughput (req/s): 2.89 \nOutput token throughput (tok/s): 614.17 \nPeak output token throughput (tok/s): 832.00 \nPeak concurrent requests: 168.00 \nTotal Token throughput (tok/s): 1202.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 14118.91 \nMedian TTFT (ms): 18607.33 \nP99 TTFT (ms): 31435.29 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 88.10 \nMedian TPOT (ms): 90.68 \nP99 TPOT (ms): 104.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 87.93 \nMedian ITL (ms): 84.69 \nP99 ITL (ms): 190.09 \n==================================================\n"
|
||||||
|
}
|
||||||
+1054
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 726.8079903329999,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 1.3758792051004176,
|
||||||
|
"tokens_per_second": 1019.986034633913
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f6885e56ca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df1894e5-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 191.60 \nTotal input tokens: 38358 \nTotal generated tokens: 39514 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 206.23 \nPeak output token throughput (tok/s): 361.00 \nPeak concurrent requests: 15.00 \nTotal Token throughput (tok/s): 406.42 \n---------------Time to First Token----------------\nMean TTFT (ms): 74.14 \nMedian TTFT (ms): 66.00 \nP99 TTFT (ms): 145.78 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 28.00 \nMedian TPOT (ms): 28.14 \nP99 TPOT (ms): 35.12 \n---------------Inter-token Latency----------------\nMean ITL (ms): 27.59 \nMedian ITL (ms): 27.66 \nP99 ITL (ms): 54.69 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7fb8a2696ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-e434292b-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 203.64 \nTotal input tokens: 146694 \nTotal generated tokens: 153347 \nRequest throughput (req/s): 3.54 \nOutput token throughput (tok/s): 753.01 \nPeak output token throughput (tok/s): 1160.00 \nPeak concurrent requests: 65.00 \nTotal Token throughput (tok/s): 1473.36 \n---------------Time to First Token----------------\nMean TTFT (ms): 93.13 \nMedian TTFT (ms): 87.23 \nP99 TTFT (ms): 181.31 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 47.81 \nMedian TPOT (ms): 47.62 \nP99 TPOT (ms): 61.13 \n---------------Inter-token Latency----------------\nMean ITL (ms): 47.43 \nMedian ITL (ms): 44.70 \nP99 ITL (ms): 109.90 \n==================================================\n"
|
||||||
|
}
|
||||||
+1076
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 449.04138348200104,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 2.2269662369327774,
|
||||||
|
"tokens_per_second": 1650.9257882903235
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f98e81ceca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-b53e2ad8-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 196.26 \nTotal input tokens: 38358 \nTotal generated tokens: 38404 \nRequest throughput (req/s): 0.92 \nOutput token throughput (tok/s): 195.68 \nPeak output token throughput (tok/s): 304.00 \nPeak concurrent requests: 18.00 \nTotal Token throughput (tok/s): 391.12 \n---------------Time to First Token----------------\nMean TTFT (ms): 519.96 \nMedian TTFT (ms): 124.31 \nP99 TTFT (ms): 10120.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 44.04 \nMedian TPOT (ms): 42.30 \nP99 TPOT (ms): 57.88 \n---------------Inter-token Latency----------------\nMean ITL (ms): 43.67 \nMedian ITL (ms): 40.54 \nP99 ITL (ms): 125.33 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4ab18f6ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-72f2dd3c-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 351.46 \nTotal input tokens: 146694 \nTotal generated tokens: 148389 \nRequest throughput (req/s): 2.05 \nOutput token throughput (tok/s): 422.20 \nPeak output token throughput (tok/s): 544.00 \nPeak concurrent requests: 369.00 \nTotal Token throughput (tok/s): 839.58 \n---------------Time to First Token----------------\nMean TTFT (ms): 72209.01 \nMedian TTFT (ms): 79268.59 \nP99 TTFT (ms): 146128.91 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 71.26 \nMedian TPOT (ms): 71.50 \nP99 TPOT (ms): 93.20 \n---------------Inter-token Latency----------------\nMean ITL (ms): 70.85 \nMedian ITL (ms): 62.03 \nP99 ITL (ms): 206.26 \n==================================================\n"
|
||||||
|
}
|
||||||
+1128
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 1089.8343978980001,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 741334,
|
||||||
|
"requests_per_second": 0.9175705978162676,
|
||||||
|
"tokens_per_second": 680.226281561525
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f532a48eca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-df39931f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 198.49 \nTotal input tokens: 37841 \nTotal generated tokens: 38900 \nRequest throughput (req/s): 0.91 \nOutput token throughput (tok/s): 195.98 \nPeak output token throughput (tok/s): 365.00 \nPeak concurrent requests: 16.00 \nTotal Token throughput (tok/s): 386.62 \n---------------Time to First Token----------------\nMean TTFT (ms): 80.34 \nMedian TTFT (ms): 69.07 \nP99 TTFT (ms): 180.64 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 37.10 \nMedian TPOT (ms): 37.08 \nP99 TPOT (ms): 40.10 \n---------------Inter-token Latency----------------\nMean ITL (ms): 37.10 \nMedian ITL (ms): 36.27 \nP99 ITL (ms): 58.47 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f081ef62ca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-ba51e591-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 719 \nFailed requests: 1 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 209.75 \nTotal input tokens: 145579 \nTotal generated tokens: 150640 \nRequest throughput (req/s): 3.43 \nOutput token throughput (tok/s): 718.18 \nPeak output token throughput (tok/s): 1259.00 \nPeak concurrent requests: 58.00 \nTotal Token throughput (tok/s): 1412.23 \n---------------Time to First Token----------------\nMean TTFT (ms): 85.92 \nMedian TTFT (ms): 75.54 \nP99 TTFT (ms): 188.35 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 42.81 \nMedian TPOT (ms): 42.63 \nP99 TPOT (ms): 54.28 \n---------------Inter-token Latency----------------\nMean ITL (ms): 42.63 \nMedian ITL (ms): 40.26 \nP99 ITL (ms): 125.13 \n==================================================\n"
|
||||||
|
}
|
||||||
+1043
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 410.0767225319996,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 736330,
|
||||||
|
"requests_per_second": 2.438568065569649,
|
||||||
|
"tokens_per_second": 1795.5908237208996
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f4b1de6aca0>, seed=0, num_prompts=180, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=1.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-073f047f-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 1.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 180 \nFailed requests: 0 \nRequest rate configured (RPS): 1.00 \nBenchmark duration (s): 190.90 \nTotal input tokens: 37841 \nTotal generated tokens: 38766 \nRequest throughput (req/s): 0.94 \nOutput token throughput (tok/s): 203.07 \nPeak output token throughput (tok/s): 413.00 \nPeak concurrent requests: 11.00 \nTotal Token throughput (tok/s): 401.29 \n---------------Time to First Token----------------\nMean TTFT (ms): 58.57 \nMedian TTFT (ms): 47.42 \nP99 TTFT (ms): 140.81 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 21.86 \nMedian TPOT (ms): 21.74 \nP99 TPOT (ms): 23.96 \n---------------Inter-token Latency----------------\nMean ITL (ms): 21.82 \nMedian ITL (ms): 21.46 \nP99 ITL (ms): 32.13 \n==================================================\n"
|
||||||
|
}
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"raw_output": "Namespace(subparser='bench', bench_type='serve', dispatch_function=<function BenchmarkServingSubcommand.cmd at 0x7f791efdeca0>, seed=0, num_prompts=720, dataset_name='sharegpt', no_stream=False, dataset_path='ShareGPT_V3_unfiltered_cleaned_split.json', no_oversample=False, skip_chat_template=False, disable_shuffle=False, custom_output_len=256, spec_bench_output_len=256, spec_bench_category=None, sonnet_input_len=550, sonnet_output_len=150, sonnet_prefix_len=200, sharegpt_output_len=None, blazedit_min_distance=0.0, blazedit_max_distance=1.0, random_input_len=1024, random_output_len=128, random_range_ratio=0.0, random_prefix_len=0, random_batch_size=1, no_reranker=False, random_mm_base_items_per_request=1, random_mm_num_mm_items_range_ratio=0.0, random_mm_limit_mm_per_prompt={'image': 255, 'video': 1}, random_mm_bucket_config={(256, 256, 1): 0.5, (720, 1280, 1): 0.5, (720, 1280, 16): 0.0}, hf_subset=None, hf_split=None, hf_name=None, hf_output_len=None, prefix_repetition_prefix_len=256, prefix_repetition_suffix_len=256, prefix_repetition_num_prefixes=10, prefix_repetition_output_len=128, label=None, backend='openai', base_url='http://127.0.0.1:8000', host='127.0.0.1', port=8000, endpoint='/v1/completions', header=None, max_concurrency=None, model='meta-llama/Meta-Llama-3.1-8B-Instruct', tokenizer=None, tokenizer_mode='auto', use_beam_search=False, logprobs=None, request_rate=4.0, burstiness=1.0, trust_remote_code=True, disable_tqdm=False, num_warmups=0, profile=False, save_result=False, save_detailed=False, append_result=False, metadata=None, result_dir=None, result_filename=None, ignore_eos=False, percentile_metrics=None, metric_percentiles='99', goodput=None, request_id_prefix='bench-2ef19409-', top_p=None, top_k=None, min_p=None, temperature=None, frequency_penalty=None, presence_penalty=None, repetition_penalty=None, common_prefix_len=None, served_model_name=None, lora_modules=None, ramp_up_strategy=None, ramp_up_start_rps=None, ramp_up_end_rps=None, ready_check_timeout_sec=600, extra_body=None)\nStarting initial single prompt test run...\nWaiting for endpoint to become up in 600 seconds\nInitial test run completed.\nStarting main benchmark run...\nTraffic request rate: 4.0\nBurstiness factor: 1.0 (Poisson process)\nMaximum request concurrency: None\ntip: install termplotlib and gnuplot to plot the metrics\n============ Serving Benchmark Result ============\nSuccessful requests: 720 \nFailed requests: 0 \nRequest rate configured (RPS): 4.00 \nBenchmark duration (s): 196.59 \nTotal input tokens: 145810 \nTotal generated tokens: 151842 \nRequest throughput (req/s): 3.66 \nOutput token throughput (tok/s): 772.38 \nPeak output token throughput (tok/s): 1274.00 \nPeak concurrent requests: 43.00 \nTotal Token throughput (tok/s): 1514.08 \n---------------Time to First Token----------------\nMean TTFT (ms): 57.73 \nMedian TTFT (ms): 49.35 \nP99 TTFT (ms): 135.16 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 25.29 \nMedian TPOT (ms): 24.90 \nP99 TPOT (ms): 34.94 \n---------------Inter-token Latency----------------\nMean ITL (ms): 25.09 \nMedian ITL (ms): 23.53 \nP99 ITL (ms): 78.49 \n==================================================\n"
|
||||||
|
}
|
||||||
+1065
File diff suppressed because it is too large
Load Diff
+7
@@ -0,0 +1,7 @@
|
|||||||
|
{
|
||||||
|
"elapsed_time": 269.04666597899995,
|
||||||
|
"num_requests": 1000,
|
||||||
|
"total_num_tokens": 736330,
|
||||||
|
"requests_per_second": 3.7168273257028708,
|
||||||
|
"tokens_per_second": 2736.811464734795
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user