cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
SZCJW
2026-03-27 00:54:43 +08:00
parent 2be354dc39
commit 6cf03e282b
5 changed files with 705 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+23 -176
View File
@@ -1,180 +1,27 @@
FROM registry.fedoraproject.org/fedora:43
FROM docker.1ms.run/kyuz0/vllm-therock-gfx1201:latest
# 1. System Base & Build Tools
# Added 'gperftools-libs' for tcmalloc (fixes double-free)
RUN dnf -y --refresh --setopt=install_weak_deps=False --setopt=metadata_expire=3600 --setopt=timeout=600 install --nodocs \
python3.13 python3.13-devel git bash ca-certificates curl \
gcc gcc-c++ binutils make cmake ninja-build \
libdrm-devel zlib-devel openssl-devel \
numactl-devel gperftools-libs dialog procps-ng \
&& dnf clean all && rm -rf /var/cache/dnf/*
# Install additional tools in a separate step
RUN dnf -y --refresh --setopt=install_weak_deps=False --setopt=metadata_expire=3600 --setopt=timeout=600 install --nodocs \
rsync libatomic ffmpeg-free aria2c tar xz vim nano jq \
&& dnf clean all && rm -rf /var/cache/dnf/*
# 2. Install "TheRock" ROCm SDK (Tarball Method)
# Note: You can pre-download the tarball to /tmp/therock.tar.gz to speed up build
WORKDIR /tmp
ARG ROCM_MAJOR_VER=7
ARG GFX=gfx120X-all
RUN if [ -f /tmp/therock.tar.gz ]; then \
echo "Using pre-downloaded tarball"; \
else \
set -euo pipefail; \
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
| tr '<' '\n' \
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
| sort -V | tail -n1)"; \
echo "Downloading Latest Tarball: ${KEY}"; \
aria2c -x 16 -s 16 -j 16 --file-allocation=none --async-dns=false \
--max-overall-download-limit=0 --max-download-limit=0 \
--min-split-size=1M --retry-wait=5 --max-tries=0 \
--timeout=600 --connect-timeout=60 \
"${BASE}/${KEY}" -o therock.tar.gz; \
fi && \
mkdir -p /opt/rocm && \
tar xzf therock.tar.gz -C /opt/rocm --strip-components=1 && \
rm therock.tar.gz
# 3. Configure Global ROCm Environment
# We add LD_PRELOAD for tcmalloc here to fix the shutdown crash
RUN export ROCM_PATH=/opt/rocm && \
BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
printf '%s\n' \
"export ROCM_PATH=/opt/rocm" \
"export HIP_PLATFORM=amd" \
"export HIP_PATH=/opt/rocm" \
"export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
"export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
"export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
"export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
"export ROCBLAS_USE_HIPBLASLT=1" \
"export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
"export VLLM_TARGET_DEVICE=rocm" \
"export HIP_FORCE_DEV_KERNARG=1" \
"export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
"export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
> /etc/profile.d/rocm-sdk.sh && \
chmod 0644 /etc/profile.d/rocm-sdk.sh
# 4. Python Venv Setup
RUN /usr/bin/python3.13 -m venv /opt/venv
ENV VIRTUAL_ENV=/opt/venv
ENV PATH=/opt/venv/bin:$PATH
ENV PIP_NO_CACHE_DIR=1
RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
# 5. Install PyTorch (TheRock Nightly)
RUN python -m pip install \
--index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
--pre torch torchaudio torchvision
# Flash-Attention
WORKDIR /opt
ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
RUN git clone https://github.com/ROCm/flash-attention.git &&\
cd flash-attention &&\
git checkout main_perf &&\
python setup.py install && \
cd /opt && rm -rf /opt/flash-attention
# 6. Clone vLLM
RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
WORKDIR /opt/vllm
# --- PATCHING ---
# vLLM relies on 'amdsmi' to detect AMD GPUs. If it's missing or fails (common in containers),
# vLLM falls back to CPU. We patch it to force ROCm detection.
RUN echo "import sys, re" > patch_vllm.py && \
echo "from pathlib import Path" >> patch_vllm.py && \
# Patch 1: __init__.py - Force is_rocm=True and bypass amdsmi checks
echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
# Patch 2: rocm.py - Mock amdsmi and force device name
echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
echo "txt = header + txt" >> patch_vllm.py && \
echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
python patch_vllm.py
# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
ENV ROCM_HOME="/opt/rocm"
ENV HIP_PATH="/opt/rocm"
ENV VLLM_TARGET_DEVICE="rocm"
ENV PYTORCH_ROCM_ARCH="gfx1201"
ENV HIP_ARCHITECTURES="gfx1201"
ENV AMDGPU_TARGETS="gfx1201"
ENV MAX_JOBS="32"
# --- FIX FOR SEGFAULT ---
# We force the Host Compiler (CC/CXX) to be the ROCm Clang, not Fedora GCC.
# This aligns the ABI of the compiled vLLM extensions with PyTorch.
ENV CC="/opt/rocm/llvm/bin/clang"
ENV CXX="/opt/rocm/llvm/bin/clang++"
RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
python -m pip install /tmp/dist/*.whl
# --- bitsandbytes (ROCm) ---
WORKDIR /opt
RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
WORKDIR /opt/bitsandbytes
# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
ENV HIP_PLATFORM="amd"
ENV CMAKE_PREFIX_PATH="/opt/rocm"
# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
RUN cmake -S . \
-DGPU_TARGETS="gfx1201" \
-DBNB_ROCM_ARCH="gfx1201" \
-DCOMPUTE_BACKEND=hip \
-DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
&& \
make -j32 && \
python -m pip install --no-cache-dir . --no-build-isolation --no-deps
# 8. Final Cleanup & Runtime
WORKDIR /opt
RUN chmod -R a+rwX /opt && \
find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
rm -rf /root/.cache/pip || true && \
dnf clean all && rm -rf /var/cache/dnf/*
COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
COPY scripts/start_vllm.py /usr/local/bin/start-vllm
COPY benchmarks/max_context_results.json /opt/max_context_results.json
COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
# Set environment variable for default model usage
# Set environment variables
ENV USE_DEFAULT_MODEL=true
ENV LOCAL_MODEL_DIR=/opt/model
# Create necessary directories
RUN mkdir -p /opt/script /opt/model /config
# Copy configuration file
COPY config.json /config/config.json
# Copy scripts to /opt/script
COPY scripts/start_vllm.py /opt/script/start-vllm
COPY benchmarks/run_vllm_bench.py /opt/script/run_vllm_bench.py
# Make scripts executable
RUN chmod +x /opt/script/start-vllm
# Create symlink for backward compatibility
RUN ln -sf /opt/script/start-vllm /usr/local/bin/start-vllm
# Set working directory
WORKDIR /opt
# Default command
CMD ["start-vllm"]
docker-compose.yml
+2 -2
View File
@@ -7,7 +7,7 @@ services:
- USE_DEFAULT_MODEL=true
- LOCAL_MODEL_DIR=/opt/model
volumes:
- /opt/project/amd-r9700-vllm-toolboxes/config.json:/config.json:ro
- /opt/project/amd-r9700-vllm-toolboxes/config.json:/config/config.json:ro
- /opt/model:/opt/model
devices:
- /dev/dri:/dev/dri
@@ -17,4 +17,4 @@ services:
- render
security_opt:
- seccomp=unconfined
restart: unless-stopped
restart: unless-stopped
scripts/run_vllm_bench.py
+356
View File
@@ -0,0 +1,356 @@
cat run_vllm_bench.py
#!/usr/bin/env python3
import subprocess, time, json, sys, os, requests, re, argparse
from pathlib import Path
# =========================
# ⚙️ GLOBAL SETTINGS
# =========================
# HARDWARE: 2x AMD Radeon AI PRO R9700 (32GB, RDNA 4)
GPU_UTIL = "0.98"
PORT = 8000
HOST = "127.0.0.1"
# BENCHMARK TOGGLES
# AITER is disabled/removed.
# 1. THROUGHPUT CONFIG
OFF_NUM_PROMPTS = 1000
OFF_FORCED_OUTPUT = "512"
# Default fallback if not specified in MODEL_TABLE
DEFAULT_BATCH_TOKENS = "8192"
# 2. LATENCY CONFIG
SRV_DURATION = 180
QPS_SWEEP = [1.0, 4.0]
# Fallbacks
FALLBACK_INPUT_LEN = 1024
FALLBACK_OUTPUT_LEN = 512
RESULTS_DIR = Path("benchmark_results")
RESULTS_DIR.mkdir(exist_ok=True)
# =========================
# 🛠️ MODEL CONFIGURATION 🛠️
# =========================
MODEL_TABLE = {
# 1. Llama 3.1 8B Instruct
# MAD uses 131k tokens. We scale to 32k for 32GB VRAM safety.
"meta-llama/Meta-Llama-3.1-8B-Instruct": {
"ctx": "65536",
"trust_remote": False,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
},
# 2. GPT-OSS 20B (MXFP4)
# MAD Row 0 uses 8192. We match this exactly.
"openai/gpt-oss-20b": {
"ctx": "32768",
"trust_remote": True,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "8192"
},
# 3. Qwen 14B FP8
# MAD uses 40k. We use 32k.
"RedHatAI/Qwen3-14B-FP8-dynamic": {
"ctx": "32768",
"trust_remote": True,
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768"
},
# 4. Qwen 30B 4-bit
"cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
"ctx": "24576",
"trust_remote": True,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
},
# 5. Qwen 80B AWQ (The Big One) [NEW]
# Size: ~48GB. Fits on 2x32GB (64GB). Leftover for Cache: ~16GB.
# Config: 20k ctx fits in that cache. Eager mode required for stability.
"cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
"ctx": "20480",
"trust_remote": True,
"valid_tp": [2], # Too big for single GPU
"max_num_seqs": "32", # Lower concurrency for safety
"max_tokens": "16384", # Lower batch size because Eager mode is CPU intensive
"enforce_eager": False,
"env": {"VLLM_USE_TRITON_AWQ": "1"} # Fixes "Unsupported Hardware" error
},
# 76 Gemma 3 27B FP8
"RedHatAI/gemma-3-27b-it-FP8-dynamic": {
"ctx": "29000",
"trust_remote": True,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "29000",
"gpu_util": "0.94",
},
# 7. Gemma 3 12B FP8
"RedHatAI/gemma-3-12b-it-FP8-dynamic": {
"ctx": "9900",
"trust_remote": True,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "9900",
},
}
MODELS_TO_RUN = [
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"openai/gpt-oss-20b",
"RedHatAI/Qwen3-14B-FP8-dynamic",
"cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit",
"cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
"RedHatAI/gemma-3-27b-it-FP8-dynamic",
"RedHatAI/gemma-3-12b-it-FP8-dynamic",
]
# =========================
# UTILS
# =========================
def log(msg): print(f"\n[BENCH] {msg}")
def get_gpu_count():
try:
# Using rocm-smi --showid to list GPUs.
# Output format: "GPU[0] : Device Name: ..."
res = subprocess.run(["rocm-smi", "--showid"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if res.returncode == 0:
# Filter specifically for the target GPU as requested
target_gpu = "AMD Radeon AI PRO R9700"
count = 0
for line in res.stdout.strip().split('\n'):
if "Device Name" in line and target_gpu in line:
count += 1
return count if count > 0 else 1
else:
log("rocm-smi failed, defaulting to 2 GPUs (Hardcoded Fallback)")
return 2
except Exception as e:
log(f"Error detecting GPUs: {e}, defaulting to 2 GPUs")
return 2
def kill_vllm():
subprocess.run("pgrep -f 'vllm serve' | xargs -r kill -9",
shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
time.sleep(5)
def nuke_vllm_cache():
cache = Path.home() / ".cache" / "vllm"
if cache.exists():
try:
subprocess.run(["rm", "-rf", str(cache)], check=True)
cache.mkdir(parents=True, exist_ok=True)
time.sleep(2)
except: pass
def get_dataset():
data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json")
if data_path.exists(): return str(data_path)
log("Downloading ShareGPT dataset...")
url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json"
try:
r = requests.get(url, stream=True, timeout=15)
r.raise_for_status()
with open(data_path, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192): f.write(chunk)
return str(data_path)
except Exception as e:
log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.")
return None
def wait_for_server(url, process, timeout=600):
start = time.time()
while time.time() - start < timeout:
if process.poll() is not None:
log(f"CRITICAL: Server died! Ret: {process.returncode}")
return False
try:
if requests.get(f"{url}/v1/models", timeout=2).status_code == 200:
log("Server ready. Stabilizing...")
time.sleep(5)
return True
except: pass
time.sleep(2)
return False
def get_model_args(model, tp_size):
config = MODEL_TABLE.get(model, {"ctx": "8192", "max_num_seqs": "32"})
# Allow per-model GPU utilization override
util = config.get("gpu_util", GPU_UTIL)
cmd = [
"--model", model,
"--gpu-memory-utilization", util,
"--max-model-len", config["ctx"],
"--dtype", "auto",
"--tensor-parallel-size", str(tp_size),
"--max-num-seqs", config["max_num_seqs"]
]
if config.get("trust_remote"): cmd.append("--trust-remote-code")
if config.get("enforce_eager"): cmd.append("--enforce-eager")
return cmd
def run_throughput(model, tp_size):
if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
model_safe = model.replace("/", "_")
output_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_throughput.json"
if output_file.exists():
log(f"SKIP Throughput {model} (TP={tp_size})")
return
dataset_path = get_dataset()
dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"]
# Retrieve Model-Specific Batch Tokens
batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS)
log(f"START Throughput {model} (TP={tp_size}) [Batch: {batch_tokens}]...")
kill_vllm()
nuke_vllm_cache()
cmd = ["vllm", "bench", "throughput"] + get_model_args(model, tp_size)
cmd.extend([
"--num-prompts", str(OFF_NUM_PROMPTS),
"--max-num-batched-tokens", batch_tokens,
"--output-len", OFF_FORCED_OUTPUT,
"--output-json", str(output_file),
"--disable-log-stats"
])
cmd.extend(dataset_args)
# ENV Setup: Global + Model Specific
env = os.environ.copy()
# Inject model specific env vars (e.g. for AWQ)
model_env = MODEL_TABLE[model].get("env", {})
env.update(model_env)
try:
subprocess.run(cmd, check=True, env=env)
except:
log(f"ERROR: Throughput failed {model}")
def run_latency(model, tp_size):
if tp_size not in MODEL_TABLE[model]["valid_tp"]: return
model_safe = model.replace("/", "_")
if all((RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{q}_latency.json").exists() for q in QPS_SWEEP):
return
dataset_path = get_dataset()
log(f"START Server {model} (TP={tp_size})...")
kill_vllm()
nuke_vllm_cache()
srv_log = open(RESULTS_DIR / f"{model_safe}_tp{tp_size}_server.log", "w")
srv_args = [x for x in get_model_args(model, tp_size) if x != "--model" and x != model]
# ENV Setup: Global + Model Specific
env = os.environ.copy()
model_env = MODEL_TABLE[model].get("env", {})
env.update(model_env)
proc = subprocess.Popen(["vllm", "serve", model] + srv_args + ["--host", HOST, "--port", str(PORT)],
stdout=srv_log, stderr=srv_log, env=env)
try:
if not wait_for_server(f"http://{HOST}:{PORT}", proc): return
for qps in QPS_SWEEP:
out_file = RESULTS_DIR / f"{model_safe}_tp{tp_size}_qps{qps}_latency.json"
if out_file.exists(): continue
log(f"BENCH QPS={qps}...")
bench_cmd = [
"vllm", "bench", "serve",
"--model", model,
"--base-url", f"http://{HOST}:{PORT}",
"--request-rate", str(qps),
"--num-prompts", str(int(max(10, SRV_DURATION * qps))),
"--trust-remote-code"
]
if dataset_path: bench_cmd.extend(["--dataset-name", "sharegpt", "--dataset-path", dataset_path])
else: bench_cmd.extend(["--dataset-name", "random", "--random-input-len", "1024", "--random-output-len", "512"])
res = subprocess.run(bench_cmd, capture_output=True, text=True, env=env)
with open(out_file, "w") as f:
f.write(json.dumps({"success": res.returncode==0, "raw_output": res.stdout}, indent=2))
except Exception as e: log(f"CRASH: {e}")
finally:
proc.terminate()
kill_vllm()
def print_summary(tps):
print(f"\n{'MODEL':<40} | {'TP':<2} | {'TOK/S':<8} | {'QPS':<4} | {'TTFT':<6} | {'TPOT':<6}")
print("-" * 105)
for m in MODELS_TO_RUN:
msafe = m.replace("/", "_")
for tp in tps:
if tp not in MODEL_TABLE[m]["valid_tp"]: continue
try:
tdata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_throughput.json").read_text())
tok_s = f"{tdata.get('tokens_per_second', 0):.1f}"
except: tok_s = "N/A"
first_row = True
for q in QPS_SWEEP:
try:
ldata = json.loads((RESULTS_DIR / f"{msafe}_tp{tp}_qps{q}_latency.json").read_text())
raw = ldata["raw_output"]
ttft = re.search(r"(?:Mean TTFT|TTFT).*?([\d\.]+)", raw).group(1)
tpot = re.search(r"(?:Mean TPOT|TPOT).*?([\d\.]+)", raw).group(1)
except: ttft, tpot = "-", "-"
name_cell = m.split('/')[-1] if (first_row and q == QPS_SWEEP[0]) else ""
print(f"{name_cell:<40} | {tp:<2} | {tok_s:<8} | {q:<4} | {ttft:<6} | {tpot:<6}")
first_row = False
print("-" * 105)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--tp", type=int, nargs="+", default=[1, 2])
args = parser.parse_args()
gpu_count = get_gpu_count()
log(f"Detected {gpu_count} AMD GPU(s)")
valid_tp_args = [t for t in args.tp if t <= gpu_count]
if not valid_tp_args:
log(f"Requested TP={args.tp} but only {gpu_count} GPU(s) detected. Nothing to run.")
sys.exit(0)
kill_vllm()
for tp in valid_tp_args:
for m in MODELS_TO_RUN:
run_throughput(m, tp)
run_latency(m, tp)
scripts/start_vllm.py
+7 -7
View File
@@ -7,18 +7,17 @@ import tempfile
import subprocess
from pathlib import Path
# Add benchmarks dir to path to import config
SCRIPT_DIR = Path(__file__).parent.resolve()
BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
# Add script dir to path
SCRIPT_DIR = Path("/opt/script")
OPT_DIR = Path("/opt")
# Config file path (check container path first, then local path)
CONFIG_PATH = Path("/config.json")
CONFIG_PATH = Path("/config/config.json")
if not CONFIG_PATH.exists():
CONFIG_PATH = SCRIPT_DIR.parent / "config.json"
CONFIG_PATH = Path("/config.json")
# Local model directory (container path)
LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", str(SCRIPT_DIR.parent / "models"))
LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", "/opt/model")
# Load configuration from config.json
try:
@@ -31,10 +30,11 @@ except Exception as e:
print(f"Error: Could not load config.json: {e}")
sys.exit(1)
# Results file path
if (OPT_DIR / "max_context_results.json").exists():
RESULTS_FILE = OPT_DIR / "max_context_results.json"
else:
RESULTS_FILE = BENCH_DIR / "max_context_results.json"
RESULTS_FILE = SCRIPT_DIR / "max_context_results.json"
HOST = os.getenv("HOST", "0.0.0.0")
PORT = os.getenv("PORT", "8000")
scripts/test_start_vllm.py
+317
View File
@@ -0,0 +1,317 @@
cat start-vllm
#!/usr/bin/env python3
import sys
import os
import json
import shutil
import tempfile
import subprocess
from pathlib import Path
# Add benchmarks dir to path to import config
# Add benchmarks dir to path to import config
SCRIPT_DIR = Path(__file__).parent.resolve()
BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
OPT_DIR = Path("/opt")
# Check /opt first (Container), then local fallback
if (OPT_DIR / "run_vllm_bench.py").exists():
sys.path.append(str(OPT_DIR))
else:
sys.path.append(str(BENCH_DIR))
try:
from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
except ImportError:
print("Error: Could not import run_vllm_bench.py config.")
sys.exit(1)
if (OPT_DIR / "max_context_results.json").exists():
RESULTS_FILE = OPT_DIR / "max_context_results.json"
else:
RESULTS_FILE = BENCH_DIR / "max_context_results.json"
HOST = os.getenv("HOST", "0.0.0.0")
PORT = os.getenv("PORT", "8000")
def check_dependencies():
if not shutil.which("dialog"):
print("Error: 'dialog' is required. Please install it (apt-get install dialog).")
sys.exit(1)
def detect_gpus():
"""Detects AMD GPUs via rocm-smi or /dev/dri."""
try:
# Try rocm-smi first
res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if res.returncode == 0:
count = res.stdout.count("GPU")
if count > 0: return count
except: pass
# Fallback to /dev/dri/render*
try:
return len(list(Path("/dev/dri").glob("renderD*")))
except:
return 1
def get_verified_config(model_id, tp_size, max_seqs):
"""
Reads max_context_results.json to find the best verified configuration.
Returns dict: {'ctx': int, 'util': float}
"""
default_config = {
"ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)),
"util": 0.90 # Safe default
}
if not RESULTS_FILE.exists():
return default_config
try:
with open(RESULTS_FILE, "r") as f:
data = json.load(f)
# Filter for Model + TP + Sequences
matches = [r for r in data
if r["model"] == model_id
and r["tp"] == tp_size
and r["max_seqs"] == max_seqs
and r["status"] == "success"]
if not matches:
# Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context?
# Actually, safer to fallback to default or try finding nearest sequence?
# Let's try finding exact match first. If fail, return default.
return default_config
# Sort by Util desc, then Context desc
# We prefer higher utilization if available (performance), as long as it is verified success
matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True)
best = matches[0]
return {
"ctx": best["max_context_1_user"],
"util": float(best["util"])
}
except Exception as e:
return default_config
def run_dialog(args):
"""Runs dialog and returns stderr (selection)."""
with tempfile.NamedTemporaryFile(mode="w+") as tf:
cmd = ["dialog"] + args
try:
subprocess.run(cmd, stderr=tf, check=True)
tf.seek(0)
return tf.read().strip()
except subprocess.CalledProcessError:
return None # User cancelled
def nuke_vllm_cache():
"""Removes vLLM cache directory to fix potential graph/incompatibility issues."""
cache = Path.home() / ".cache" / "vllm"
if cache.exists():
try:
print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
subprocess.run(["rm", "-rf", str(cache)], check=True)
cache.mkdir(parents=True, exist_ok=True)
print(" Done.")
time.sleep(1)
except Exception as e:
print(f" Failed: {e}")
def configure_and_launch(model_idx, gpu_count):
model_id = MODELS_TO_RUN[model_idx]
config = MODEL_TABLE[model_id]
# Static Config
valid_tps = config.get("valid_tp", [1])
max_tp = max(valid_tps) if valid_tps else 1
# Defaults
current_tp = min(gpu_count, max_tp)
current_seqs = 1 # Default to 1 concurrent user/request for stability
# Initial Lookup
verified = get_verified_config(model_id, current_tp, current_seqs)
current_ctx = verified["ctx"]
current_util = verified["util"]
clear_cache = False
use_eager = config.get("enforce_eager", False) # Default to model config, usually False
use_rocm_attn = False # Default to Triton
name = model_id.split("/")[-1]
while True:
cache_status = "YES" if clear_cache else "NO"
eager_status = "YES" if use_eager else "NO"
attn_backend = "ROCm" if use_rocm_attn else "Triton"
menu_args = [
"--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
"--title", f"Configuration: {name}",
"--menu", "Customize Launch Parameters:", "22", "65", "9",
"1", f"Tensor Parallelism: {current_tp}",
"2", f"Concurrent Requests: {current_seqs}",
"3", f"Context Length: {current_ctx} (Verified)",
"4", f"GPU Utilization: {current_util} (Verified)",
"5", f"Attention Backend: {attn_backend}",
"6", f"Erase vLLM Cache: {cache_status}",
"7", f"Force Eager Mode: {eager_status}",
"8", "LAUNCH SERVER"
]
choice = run_dialog(menu_args)
if not choice: return False # Back/Cancel
if choice == "1":
# TP Selection
new_tp = run_dialog([
"--title", "Tensor Parallelism",
"--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp)
])
if new_tp:
new_tp_int = int(new_tp)
if new_tp_int != current_tp:
current_tp = new_tp_int
# RE-CALCULATE Config
verified = get_verified_config(model_id, current_tp, current_seqs)
current_ctx = verified["ctx"]
current_util = verified["util"]
elif choice == "2":
# Max Seqs Selection
new_seqs = run_dialog([
"--title", "Concurrent Requests",
"--menu", "Select Max Concurrent Requests:", "12", "40", "4",
"1", "1 (Latency Focus)",
"4", "4 (Balanced)",
"8", "8 (Throughput)",
"16", "16 (Max Load)"
])
if new_seqs:
current_seqs = int(new_seqs)
# RE-CALCULATE Config based on new concurrency
verified = get_verified_config(model_id, current_tp, current_seqs)
current_ctx = verified["ctx"]
current_util = verified["util"]
elif choice == "3":
# Configured Length Override
new_ctx = run_dialog([
"--title", "Context Length",
"--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx)
])
if new_ctx: current_ctx = int(new_ctx)
elif choice == "4":
# Util Override
pass
elif choice == "5":
# Toggle Attention Backend
use_rocm_attn = not use_rocm_attn
elif choice == "6":
# Toggle Cache
if not clear_cache:
# Enabling it -> Show Warning
warn_msg = (
"WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n"
"This is useful if you are experiencing crashes, 'invalid graph' errors,\n"
"or have switched vLLM versions recently.\n\n"
"However, the next startup will take longer as graphs are re-compiled.\n\n"
"Are you sure you want to enable this?"
)
confirm = run_dialog([
"--title", "Erase Cache Warning",
"--yesno", warn_msg, "12", "60"
])
# If confirm is not None (exit 0), it is YES.
if confirm is not None:
clear_cache = True
else:
# Disabling it -> No warning needed
clear_cache = False
elif choice == "7":
# Toggle Eager Mode
use_eager = not use_eager
elif choice == "8":
# Launch
break
# Build Command
subprocess.run(["clear"])
if clear_cache:
nuke_vllm_cache()
cmd = [
"vllm", "serve", model_id,
"--host", HOST,
"--port", PORT,
"--tensor-parallel-size", str(current_tp),
"--max-num-seqs", str(current_seqs),
"--max-model-len", str(current_ctx),
"--gpu-memory-utilization", str(current_util),
"--dtype", "auto"
]
if config.get("trust_remote"): cmd.append("--trust-remote-code")
if use_eager: cmd.append("--enforce-eager")
# Env Vars
env = os.environ.copy()
env.update(config.get("env", {}))
if use_rocm_attn:
env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
# Optional: Explicitly mention these in print
print("\n" + "="*60)
print(f" Launching: {name}")
print(f" Config: TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}")
print(f" Backend: {'ROCm' if use_rocm_attn else 'Triton'}")
if clear_cache:
print(f" Action: Clearing vLLM Cache (~/.cache/vllm)")
print(f" Command: {' '.join(cmd)}")
print("="*60 + "\n")
os.execvpe("vllm", cmd, env)
def main():
check_dependencies()
gpu_count = detect_gpus()
while True:
# Build Model Menu
menu_items = []
for i, m_id in enumerate(MODELS_TO_RUN):
name = m_id.split("/")[-1]
# Pre-calc verified ctx for 'default' TP to show in menu?
# Or just show names. Just names is cleaner.
config = MODEL_TABLE[m_id]
menu_items.extend([str(i), name])
choice = run_dialog([
"--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
"--title", "Select Model",
"--menu", "Choose a model to serve:", "20", "60", "10"
] + menu_items)
if not choice:
subprocess.run(["clear"])
print("Selection cancelled.")
sys.exit(0)
configure_and_launch(int(choice), gpu_count)
if __name__ == "__main__":
main()