cjw/vllm-r9700-container
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
SZCJW
2026-03-10 23:09:05 +08:00
parent 1c08c96d32
commit 7a38199dde
16 changed files with 1702 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/.gitignore
Generated
+5
View File
@@ -0,0 +1,5 @@
# 默认忽略的文件
/shelf/
/workspace.xml
# 基于编辑器的 HTTP 客户端请求
/httpRequests/
.idea/inspectionProfiles/Project_Default.xml
Generated
+13
View File
@@ -0,0 +1,13 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<list>
<option value="python-jose" />
<option value="passlib" />
</list>
</option>
</inspection_tool>
</profile>
</component>
.idea/inspectionProfiles/profiles_settings.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
.idea/misc.xml
Generated
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.13 (base)" project-jdk-type="Python SDK" />
</project>
.idea/modules.xml
Generated
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/vllm-r9700-container.iml" filepath="$PROJECT_DIR$/.idea/vllm-r9700-container.iml" />
</modules>
</component>
</project>
.idea/vcs.xml
Generated
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
.idea/vllm-r9700-container.iml
Generated
+12
View File
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.13 (base)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>
Dockerfile
+172
View File
@@ -0,0 +1,172 @@
FROM registry.fedoraproject.org/fedora:43
# 1. System Base & Build Tools
RUN dnf -y install --setopt=install_weak_deps=False --nodocs \
python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \
gcc gcc-c++ binutils make ffmpeg-free \
cmake ninja-build aria2c tar xz vim nano \
libdrm-devel zlib-devel openssl-devel jq \
numactl-devel gperftools-libs procps-ng \
&& dnf clean all && rm -rf /var/cache/dnf/*
# 2. Install "TheRock" ROCm SDK (Tarball Method)
WORKDIR /tmp
ARG ROCM_MAJOR_VER=7
ARG GFX=gfx120X-all
RUN set -euo pipefail; \
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
| tr '<' '\n' \
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
| sort -V | tail -n1)"; \
echo "Downloading Latest Tarball: ${KEY}"; \
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \
mkdir -p /opt/rocm; \
tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \
rm therock.tar.gz
# 3. Configure Global ROCm Environment
RUN export ROCM_PATH=/opt/rocm && \
BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
printf '%s\n' \
"export ROCM_PATH=/opt/rocm" \
"export HIP_PLATFORM=amd" \
"export HIP_PATH=/opt/rocm" \
"export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \
"export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \
"export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \
"export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \
"export ROCBLAS_USE_HIPBLASLT=1" \
"export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \
"export VLLM_TARGET_DEVICE=rocm" \
"export HIP_FORCE_DEV_KERNARG=1" \
"export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \
"export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \
> /etc/profile.d/rocm-sdk.sh && \
chmod 0644 /etc/profile.d/rocm-sdk.sh
# 4. Python Venv Setup
RUN /usr/bin/python3.13 -m venv /opt/venv
ENV VIRTUAL_ENV=/opt/venv
ENV PATH=/opt/venv/bin:$PATH
ENV PIP_NO_CACHE_DIR=1
RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh
RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0"
# 5. Install PyTorch (TheRock Nightly) and PyYAML
RUN python -m pip install \
--index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \
--pre torch torchaudio torchvision && \
python -m pip install pyyaml
# Flash-Attention
WORKDIR /opt
ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
RUN git clone https://github.com/ROCm/flash-attention.git &&\ \
cd flash-attention &&\
git checkout main_perf &&\
python setup.py install && \
cd /opt && rm -rf /opt/flash-attention
# 6. Clone vLLM
RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm
WORKDIR /opt/vllm
# --- PATCHING ---
RUN echo "import sys, re" > patch_vllm.py && \
echo "from pathlib import Path" >> patch_vllm.py && \
echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \
echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \
echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \
echo "txt = p.read_text()" >> patch_vllm.py && \
echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \
echo "txt = header + txt" >> patch_vllm.py && \
echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \
echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \
echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \
echo "p.write_text(txt)" >> patch_vllm.py && \
echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \
python patch_vllm.py
# 7. Build vLLM (Wheel Method) with CLANG Host Compiler
RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11
ENV ROCM_HOME="/opt/rocm"
ENV HIP_PATH="/opt/rocm"
ENV VLLM_TARGET_DEVICE="rocm"
ENV PYTORCH_ROCM_ARCH="gfx1201"
ENV HIP_ARCHITECTURES="gfx1201"
ENV AMDGPU_TARGETS="gfx1201"
ENV MAX_JOBS="4"
# --- FIX FOR SEGFAULT ---
ENV CC="/opt/rocm/llvm/bin/clang"
ENV CXX="/opt/rocm/llvm/bin/clang++"
RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \
echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \
export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \
python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \
python -m pip install /tmp/dist/*.whl
# --- bitsandbytes (ROCm) ---
WORKDIR /opt
RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git
WORKDIR /opt/bitsandbytes
# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile)
ENV HIP_PLATFORM="amd"
ENV CMAKE_PREFIX_PATH="/opt/rocm"
# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++)
RUN cmake -S . \
-DGPU_TARGETS="gfx1201" \
-DBNB_ROCM_ARCH="gfx1201" \
-DCOMPUTE_BACKEND=hip \
-DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
-DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \
&& \
make -j$(nproc) && \
python -m pip install --no-cache-dir . --no-build-isolation --no-deps
# 8. Final Cleanup & Runtime
WORKDIR /opt
RUN chmod -R a+rwX /opt && \
find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \
find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \
rm -rf /root/.cache/pip || true && \
dnf clean all && rm -rf /var/cache/dnf/*
# Create vLLM configuration directory
RUN mkdir -p /etc/vllm
# Create default YAML configuration file
RUN printf '# model_config.yaml - 多模型配置\n# 默认启动的模型\ndefault: "deepseek_r1_distill_qwen_32b_awq"\n\n# 模型配置\nmodels:\n deepseek_r1_distill_qwen_14b:\n path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B"\n name: "DeepSeek-R1-Distill-Qwen-14B"\n max_model_len: 8192\n gpu_memory_utilization: 0.9\n port: 2001\n dtype: "float16"\n quantization: "awq"\n tensor_parallel_size: 1\n enforce_eager: true\n api_key: "sk-14b-20240101-abcdef123456"\n \n deepseek_r1_distill_qwen_32b_awq:\n path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ"\n name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"\n \n # 模型性能参数\n max_model_len: 32768\n gpu_memory_utilization: 0.95\n enforce_eager: true\n max_num_seqs: 2\n max_num_batched_tokens: 1024\n block_size: 16\n tensor_parallel_size: 1\n swap_space: 0\n \n # 新增：采样参数默认值\n sampling_defaults:\n temperature: 0.6\n max_tokens: 4096\n top_p: 0.9\n frequency_penalty: 0.0\n presence_penalty: 0.0\n stop:\n - "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
\n # 其他配置\n dtype: "auto"\n quantization: "awq"\n port: 2001\n api_key: "sk-32b-20240101-ghijk789012"\n \n glm_4_7_flash_awq:\n path: "/app/vllm/models/GLM-4.7-Flash-AWQ"\n name: "GLM-4.7-Flash-AWQ"\n \n # 模型性能参数\n max_model_len: 32768\n gpu_memory_utilization: 0.9\n enforce_eager: false\n max_num_seqs: 2\n max_num_batched_tokens: 1024\n block_size: 16\n tensor_parallel_size: 1\n swap_space: 0\n \n # 新增：采样参数默认值\n sampling_defaults:\n temperature: 0.6\n max_tokens: 4096\n top_p: 0.9\n frequency_penalty: 0.0\n presence_penalty: 0.0\n stop:\n - "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
\n # 其他配置\n dtype: "auto"\n quantization: "awq"\n port: 2001\n api_key: "sk-32b-20240101-ghijk789012"\n \n qwen3_vl_32b_instruct_awq:\n path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ"\n name: "Qwen3-VL-32B-Instruct-AWQ"\n max_model_len: 32768\n gpu_memory_utilization: 0.7\n port: 2001\n dtype: "auto"\n quantization: "awq"\n tensor_parallel_size: 1\n enforce_eager: true\n api_key: "sk-glm-20240101-lmnop345678"\n\n# 服务器通用设置\nserver:\n host: "0.0.0.0"\n log_level: "info"\n # 全局管理员密钥（拥有所有模型的访问权限）\n admin_key: "sk-admin-20240101-xyz789"\n # 允许的请求头名称（支持多个，按顺序检查）\n api_key_headers: ["Authorization", "X-API-Key", "api-key"]\n # 是否允许通过查询参数传递密钥\n allow_query_param: true\n # 查询参数名称\n api_key_param: "api_key"' > /etc/vllm/model_config.yaml
# Copy necessary scripts
COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh
COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh
COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh
COPY scripts/start_vllm.py /usr/local/bin/start-vllm
RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm
RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
CMD ["/bin/bash"]
README.md
+443 -1
View File
@@ -1,2 +1,444 @@
# vllm-r9700-container # 自定义 vLLM 容器
这是一个基于 Fedora 的 Docker/Podman 容器，专为在 AMD Radeon R9700 (gfx1201) GPU 上运行 vLLM 而设计。
## 特性
- 基于 Fedora 43
- 使用最新的 TheRock ROCm 7.x SDK
- 包含 PyTorch 预发布版本（ROCm 支持）
- 内置 Flash-Attention（ROCm 版本）
- 支持多种大型语言模型
- 基于配置文件的启动方式
- 仅支持本地模型（无网络下载功能）
- 多模型配置支持
- **提供标准的 OpenAI API 服务**
## 快速开始
### 1. 构建并运行（推荐）
```bash
# 赋予脚本执行权限
chmod +x build_and_run.sh
# 一键构建并运行
./build_and_run.sh
```
### 2. 手动构建和运行
```bash
# 构建镜像
docker build -t custom-vllm-r9700:latest .
# 运行容器
docker run -it --device /dev/dri --device /dev/kfd \
--group-add video --group-add render --security-opt seccomp=unconfined \
-v /path/to/models:/models \
-v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \
-e LOCAL_MODEL_DIR=/models \
custom-vllm-r9700:latest
```
## 构建容器
在项目目录中运行：
```bash
docker build -t custom-vllm-r9700:latest .
```
## 使用方法
### 使用 Docker/Podman
```bash
docker run -it --device /dev/dri --device /dev/kfd \
--group-add video --group-add render --security-opt seccomp=unconfined \
-v /path/to/models:/models \
-v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \
-e LOCAL_MODEL_DIR=/models \
custom-vllm-r9700:latest
```
### 使用 Toolbx（Fedora）
```bash
toolbox create vllm-custom \
--image custom-vllm-r9700:latest \
-- --device /dev/dri --device /dev/kfd \
--group-add video --group-add render --security-opt seccomp=unconfined
toolbox enter vllm-custom
```
### 使用 Distrobox（Ubuntu）
```bash
distrobox create -n vllm-custom \
--image custom-vllm-r9700:latest \
--additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined"
distrobox enter vllm-custom
```
## 配置文件
容器使用 YAML 格式的配置文件来设置 vLLM 服务器参数。默认配置文件位于 `/etc/vllm/model_config.yaml`。项目根目录中提供了配置文件示例 `model_config.yaml.example`，您可以参考它来创建自己的配置文件。
### 配置文件示例
```yaml
# model_config.yaml - 多模型配置示例
# 默认启动的模型
default: "deepseek_r1_distill_qwen_32b_awq"
# 模型配置
models:
deepseek_r1_distill_qwen_14b:
path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B"
name: "DeepSeek-R1-Distill-Qwen-14B"
max_model_len: 8192
gpu_memory_utilization: 0.9
port: 2001
dtype: "float16"
quantization: "awq"
tensor_parallel_size: 1
enforce_eager: true
api_key: "sk-14b-20240101-abcdef123456"
deepseek_r1_distill_qwen_32b_awq:
path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ"
name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"
# 模型性能参数
max_model_len: 32768
gpu_memory_utilization: 0.95
enforce_eager: true
max_num_seqs: 2
max_num_batched_tokens: 1024
block_size: 16
tensor_parallel_size: 1
swap_space: 0
# 新增：采样参数默认值
sampling_defaults:
temperature: 0.6
max_tokens: 4096
top_p: 0.9
frequency_penalty: 0.0
presence_penalty: 0.0
stop:
- "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
# 其他配置
dtype: "auto"
quantization: "awq"
port: 2001
api_key: "sk-32b-20240101-ghijk789012"
glm_4_7_flash_awq:
path: "/app/vllm/models/GLM-4.7-Flash-AWQ"
name: "GLM-4.7-Flash-AWQ"
# 模型性能参数
max_model_len: 32768
gpu_memory_utilization: 0.9
enforce_eager: false
max_num_seqs: 2
max_num_batched_tokens: 1024
block_size: 16
tensor_parallel_size: 1
swap_space: 0
# 新增：采样参数默认值
sampling_defaults:
temperature: 0.6
max_tokens: 4096
top_p: 0.9
frequency_penalty: 0.0
presence_penalty: 0.0
stop:
- "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
# 其他配置
dtype: "auto"
quantization: "awq"
port: 2001
api_key: "sk-32b-20240101-ghijk789012"
qwen3_vl_32b_instruct_awq:
path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ"
name: "Qwen3-VL-32B-Instruct-AWQ"
max_model_len: 32768
gpu_memory_utilization: 0.7
port: 2001
dtype: "auto"
quantization: "awq"
tensor_parallel_size: 1
enforce_eager: true
api_key: "sk-glm-20240101-lmnop345678"
# 服务器通用设置
server:
host: "0.0.0.0"
log_level: "info"
# 全局管理员密钥（拥有所有模型的访问权限）
admin_key: "sk-admin-20240101-xyz789"
# 允许的请求头名称（支持多个，按顺序检查）
api_key_headers: ["Authorization", "X-API-Key", "api-key"]
# 是否允许通过查询参数传递密钥
allow_query_param: true
# 查询参数名称
api_key_param: "api_key"
```
### 配置参数说明
- `default`：默认启动的模型名称
- `models`：模型配置列表
- 每个模型包含：
- `path`：模型路径
- `name`：模型名称
- `max_model_len`：最大模型上下文长度
- `gpu_memory_utilization`：GPU 内存利用率
- `port`：服务器端口
- `dtype`：数据类型
- `quantization`：量化方式
- `tensor_parallel_size`：张量并行度
- `enforce_eager`：是否强制使用 eager 模式
- `max_num_seqs`：最大并发请求数
- `max_num_batched_tokens`：最大批量 tokens 数
- `block_size`：块大小
- `swap_space`：交换空间大小
- `sampling_defaults`：采样参数默认值
- `api_key`：API 密钥（用于 OpenAI 兼容模式）
- `server`：服务器通用设置
- `host`：服务器主机地址
- `log_level`：日志级别
- `admin_key`：全局管理员密钥
- `api_key_headers`：允许的请求头名称
- `allow_query_param`：是否允许通过查询参数传递密钥
- `api_key_param`：查询参数名称
## 环境变量
- `LOCAL_MODEL_DIR`：本地模型目录路径（必须设置）
- `VLLM_CONFIG_FILE`：配置文件路径（默认：/etc/vllm/model_config.yaml）
## 启动 vLLM 服务器
### 使用构建和部署脚本（推荐）
项目提供了 `build_and_run.sh` 脚本，用于一键构建镜像并部署运行：
```bash
# 赋予执行权限
chmod +x build_and_run.sh
# 构建并运行（默认后台运行）
./build_and_run.sh
# 仅构建镜像
./build_and_run.sh -b
# 重新构建镜像（不使用缓存）
./build_and_run.sh -r
# 仅停止并删除容器
./build_and_run.sh -s
# 指定端口运行
./build_and_run.sh -p 8080
# 指定配置文件和模型目录
./build_and_run.sh -c /path/to/config.yaml -m /path/to/models
# 交互式运行（前台运行）
./build_and_run.sh -i
```
**脚本选项：**
- `-b, --build-only`：仅构建镜像，不运行容器
- `-r, --rebuild`：重新构建镜像（不使用缓存）
- `-s, --stop`：仅停止并删除容器
- `-d, --detach`：后台运行容器（默认）
- `-i, --interactive`：交互式运行容器
- `-c, --config FILE`：指定配置文件路径
- `-m, --models DIR`：指定模型目录路径
- `-p, --port PORT`：指定服务端口
- `-h, --help`：显示帮助信息
### 手动部署
```bash
# 1. 构建镜像
docker build -t custom-vllm-r9700:latest .
# 2. 运行容器
docker run -it --device /dev/dri --device /dev/kfd \
--group-add video --group-add render --security-opt seccomp=unconfined \
-v /path/to/models:/models \
-v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \
-e LOCAL_MODEL_DIR=/models \
custom-vllm-r9700:latest
```
### 容器内运行
进入容器后，可以使用以下命令启动 vLLM 服务器：
```bash
# 使用默认模型启动
start-vllm
# 或指定模型名称启动
start-vllm deepseek_r1_distill_qwen_14b
# 或直接指定模型路径
vllm serve /models/model-name --tensor-parallel-size 2 --max-model-len 128000
```
## 测试 API（OpenAI 兼容）
vLLM 提供与 OpenAI API 完全兼容的服务接口。
### 1. 使用 curl 测试
```bash
# 聊天补全接口
curl -X POST http://localhost:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-32b-20240101-ghijk789012" \
-d '{
"model": "deepseek_r1_distill_qwen_32b_awq",
"messages": [
{"role": "user", "content": "你好，请介绍一下你自己"}
],
"temperature": 0.6,
"max_tokens": 4096,
"top_p": 0.9
}'
# 文本补全接口
curl -X POST http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-32b-20240101-ghijk789012" \
-d '{
"model": "deepseek_r1_distill_qwen_32b_awq",
"prompt": "Once upon a time",
"max_tokens": 100
}'
# 列出可用模型
curl http://localhost:8000/v1/models \
-H "Authorization: Bearer sk-32b-20240101-ghijk789012"
```
### 2. 使用 Python OpenAI SDK
```python
from openai import OpenAI
# 初始化客户端
client = OpenAI(
base_url="http://localhost:8000/v1",
api_key="sk-32b-20240101-ghijk789012"
)
# 聊天补全
response = client.chat.completions.create(
model="deepseek_r1_distill_qwen_32b_awq",
messages=[
{"role": "user", "content": "你好，请介绍一下你自己"}
],
temperature=0.6,
max_tokens=4096
)
print(response.choices[0].message.content)
# 文本补全
response = client.completions.create(
model="deepseek_r1_distill_qwen_32b_awq",
prompt="Once upon a time",
max_tokens=100
)
print(response.choices[0].text)
```
### 3. 使用其他 OpenAI 兼容工具
由于提供标准的 OpenAI API，您可以使用任何支持 OpenAI 的工具和库，例如：
- LangChain
- LlamaIndex
- AutoGen
- FastChat
- 等等
只需将 `base_url` 设置为 `http://localhost:8000/v1`，并使用配置的 API 密钥即可。
## 本地模型目录结构
确保本地模型目录包含以下文件之一：
- `config.json`
- `pytorch_model.bin`
- `model.safetensors`
正确的目录结构示例：
```
/models/
├── deepseek_r1_distill_qwen_14b/
│ ├── config.json
│ └── model.safetensors
├── deepseek_r1_distill_qwen_32b_awq/
│ ├── config.json
│ └── pytorch_model.bin
├── glm_4_7_flash_awq/
│ ├── config.json
│ └── model.safetensors
└── qwen3_vl_32b_instruct_awq/
├── config.json
└── pytorch_model.bin
```
## 注意事项
- 确保您的 AMD Radeon R9700 GPU 驱动已正确安装
- 容器需要访问 GPU 设备，因此运行时需要添加 `--device /dev/dri --device /dev/kfd` 参数
- 首次启动时，vLLM 会编译计算图，可能需要较长时间
- 如果遇到内存不足的问题，可以调整 `gpu_memory_utilization` 参数
- API 密钥在配置文件的每个模型中单独配置，用于 OpenAI 兼容模式的认证
## OpenAI API 兼容性
本容器提供的服务完全兼容 OpenAI API 标准，包括：
- **聊天补全**：`/v1/chat/completions`
- **文本补全**：`/v1/completions`
- **模型列表**：`/v1/models`
- **嵌入**：`/v1/embeddings`（如支持）
所有端点都支持标准的 OpenAI 请求格式和参数，您可以无缝切换使用。
## 项目文件说明
- `Dockerfile` - Docker 镜像构建文件
- `README.md` - 项目说明文档
- `model_config.yaml.example` - 配置文件示例
- `params_config.yaml` - 参数配置文件（定义各类参数的类型和验证规则）
- `build_and_run.sh` - 构建和部署脚本
- `scripts/start_vllm.py` - vLLM 启动脚本
build_and_run.sh
+291
View File
@@ -0,0 +1,291 @@
#!/bin/bash
# vLLM R9700 Container 构建和部署脚本
# 用于一键构建镜像并部署运行
set -e
# 配置变量
IMAGE_NAME="custom-vllm-r9700"
IMAGE_TAG="latest"
CONTAINER_NAME="custom-vllm-r9700"
MODEL_DIR="${MODEL_DIR:-/opt/models}"
CONFIG_FILE="${CONFIG_FILE:-/opt/model_config.yaml}"
LOCAL_MODEL_DIR="${LOCAL_MODEL_DIR:-/models}"
# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# 打印带颜色的消息
print_info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
print_success() {
echo -e "${GREEN}[SUCCESS]${NC} $1"
}
print_warning() {
echo -e "${YELLOW}[WARNING]${NC} $1"
}
print_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# 显示使用帮助
show_usage() {
echo "用法：$0 [选项]"
echo ""
echo "选项:"
echo " -b, --build-only 仅构建镜像，不运行容器"
echo " -r, --rebuild 重新构建镜像（不使用缓存）"
echo " -s, --stop 仅停止并删除容器"
echo " -d, --detach 后台运行容器（默认）"
echo " -i, --interactive 交互式运行容器"
echo " -c, --config FILE 指定配置文件路径"
echo " -m, --models DIR 指定模型目录路径"
echo " -p, --port PORT 指定服务端口"
echo " -h, --help 显示此帮助信息"
echo ""
echo "示例:"
echo " $0 # 构建并运行"
echo " $0 -b # 仅构建镜像"
echo " $0 -r # 重新构建并运行"
echo " $0 -s # 停止并删除容器"
echo " $0 -p 8080 # 使用端口 8080 运行"
echo " $0 -c /path/to/config.yaml # 使用指定配置文件"
echo " $0 -m /path/to/models # 使用指定模型目录"
exit 0
}
# 解析命令行参数
BUILD_ONLY=false
REBUILD=false
STOP_ONLY=false
DETACH=true
CUSTOM_CONFIG=""
CUSTOM_MODELS=""
CUSTOM_PORT=""
while [[ $# -gt 0 ]]; do
case $1 in
-b|--build-only)
BUILD_ONLY=true
shift
;;
-r|--rebuild)
REBUILD=true
shift
;;
-s|--stop)
STOP_ONLY=true
shift
;;
-d|--detach)
DETACH=true
shift
;;
-i|--interactive)
DETACH=false
shift
;;
-c|--config)
CUSTOM_CONFIG="$2"
shift 2
;;
-m|--models)
CUSTOM_MODELS="$2"
shift 2
;;
-p|--port)
CUSTOM_PORT="$2"
shift 2
;;
-h|--help)
show_usage
;;
*)
print_error "未知选项：$1"
show_usage
;;
esac
done
# 停止并删除容器
stop_container() {
print_info "停止并删除容器..."
if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
docker stop "${CONTAINER_NAME}" 2>/dev/null || true
print_success "容器已停止"
docker rm "${CONTAINER_NAME}" 2>/dev/null || true
print_success "容器已删除"
else
print_warning "容器不存在"
fi
}
# 如果只需要停止容器，直接退出
if [ "$STOP_ONLY" = true ]; then
stop_container
exit 0
fi
# 检查 Docker 是否运行
check_docker() {
if ! docker info > /dev/null 2>&1; then
print_error "Docker 未运行或无权限访问"
print_error "请确保 Docker 服务正在运行，并且当前用户有 Docker 访问权限"
exit 1
fi
print_success "Docker 检查通过"
}
# 构建镜像
build_image() {
print_info "开始构建 Docker 镜像..."
print_info "镜像名称：${IMAGE_NAME}:${IMAGE_TAG}"
BUILD_ARGS="--no-cache"
if [ "$REBUILD" = false ]; then
BUILD_ARGS=""
fi
if docker build ${BUILD_ARGS} -t "${IMAGE_NAME}:${IMAGE_TAG}" .; then
print_success "镜像构建成功"
docker images "${IMAGE_NAME}:${IMAGE_TAG}" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}"
else
print_error "镜像构建失败"
exit 1
fi
}
# 运行容器
run_container() {
print_info "开始部署容器..."
# 设置卷挂载参数
VOLUME_ARGS=""
# 模型目录
if [ -n "$CUSTOM_MODELS" ]; then
MODEL_DIR="$CUSTOM_MODELS"
fi
if [ -d "$MODEL_DIR" ]; then
VOLUME_ARGS="$VOLUME_ARGS -v ${MODEL_DIR}:${LOCAL_MODEL_DIR}"
print_info "模型目录：${MODEL_DIR} -> ${LOCAL_MODEL_DIR}"
else
print_warning "模型目录不存在：${MODEL_DIR}"
print_info "创建模型目录..."
mkdir -p "$MODEL_DIR"
VOLUME_ARGS="$VOLUME_ARGS -v ${MODEL_DIR}:${LOCAL_MODEL_DIR}"
fi
# 配置文件
if [ -n "$CUSTOM_CONFIG" ]; then
CONFIG_FILE="$CUSTOM_CONFIG"
fi
if [ -f "$CONFIG_FILE" ]; then
VOLUME_ARGS="$VOLUME_ARGS -v ${CONFIG_FILE}:/etc/vllm/model_config.yaml"
print_info "配置文件：${CONFIG_FILE}"
else
print_warning "配置文件不存在：${CONFIG_FILE}"
print_info "将使用容器内默认配置"
fi
# 端口设置
PORT_ARG="-p 8000:8000"
if [ -n "$CUSTOM_PORT" ]; then
PORT_ARG="-p ${CUSTOM_PORT}:8000"
fi
# 运行模式
RUN_MODE="-d"
if [ "$DETACH" = false ]; then
RUN_MODE="-it"
fi
# 构建运行命令
print_info "启动容器..."
docker run ${RUN_MODE} \
--name "${CONTAINER_NAME}" \
--device /dev/dri \
--device /dev/kfd \
--group-add video \
--group-add render \
--security-opt seccomp=unconfined \
${VOLUME_ARGS} \
${PORT_ARG} \
-e LOCAL_MODEL_DIR="${LOCAL_MODEL_DIR}" \
"${IMAGE_NAME}:${IMAGE_TAG}"
if [ $? -eq 0 ]; then
print_success "容器启动成功"
if [ "$DETACH" = true ]; then
echo ""
print_info "容器信息:"
echo " 容器名称：${CONTAINER_NAME}"
echo " 镜像：${IMAGE_NAME}:${IMAGE_TAG}"
echo " 状态：运行中"
echo " API 地址：http://localhost:8000"
echo ""
print_info "常用命令:"
echo " 查看日志：docker logs -f ${CONTAINER_NAME}"
echo " 停止容器：docker stop ${CONTAINER_NAME}"
echo " 启动容器：docker start ${CONTAINER_NAME}"
echo " 删除容器：docker rm -f ${CONTAINER_NAME}"
echo " 进入容器：docker exec -it ${CONTAINER_NAME} /bin/bash"
fi
else
print_error "容器启动失败"
exit 1
fi
}
# 主流程
main() {
echo "=========================================="
echo "vLLM R9700 Container 构建和部署"
echo "=========================================="
echo "开始时间：$(date '+%Y-%m-%d %H:%M:%S')"
echo ""
# 检查 Docker
check_docker
# 停止旧容器
stop_container
# 构建镜像
build_image
# 如果仅构建，退出
if [ "$BUILD_ONLY" = true ]; then
print_success "镜像构建完成"
print_info "运行以下命令启动容器:"
echo " $0"
exit 0
fi
# 运行容器
run_container
echo ""
echo "=========================================="
print_success "部署完成"
echo "=========================================="
echo "完成时间：$(date '+%Y-%m-%d %H:%M:%S')"
}
# 执行主流程
main
model_config.yaml.example
+110
View File
@@ -0,0 +1,110 @@
# model_config.yaml - 多模型配置示例
# 默认启动的模型
default: "deepseek_r1_distill_qwen_32b_awq"
# 模型配置
models:
deepseek_r1_distill_qwen_14b:
path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B"
name: "DeepSeek-R1-Distill-Qwen-14B"
max_model_len: 8192
gpu_memory_utilization: 0.9
port: 2001
dtype: "float16"
quantization: "awq"
tensor_parallel_size: 1
enforce_eager: true
api_key: "sk-14b-20240101-abcdef123456"
deepseek_r1_distill_qwen_32b_awq:
path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ"
name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"
# 模型性能参数
max_model_len: 32768
gpu_memory_utilization: 0.95
enforce_eager: true
max_num_seqs: 2
max_num_batched_tokens: 1024
block_size: 16
tensor_parallel_size: 1
swap_space: 0
# 新增：采样参数默认值
sampling_defaults:
temperature: 0.6
max_tokens: 4096
top_p: 0.9
frequency_penalty: 0.0
presence_penalty: 0.0
stop:
- "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
# 其他配置
dtype: "auto"
quantization: "awq"
port: 2001
api_key: "sk-32b-20240101-ghijk789012"
glm_4_7_flash_awq:
path: "/app/vllm/models/GLM-4.7-Flash-AWQ"
name: "GLM-4.7-Flash-AWQ"
# 模型性能参数
max_model_len: 32768
gpu_memory_utilization: 0.9
enforce_eager: false
max_num_seqs: 2
max_num_batched_tokens: 1024
block_size: 16
tensor_parallel_size: 1
swap_space: 0
# 新增：采样参数默认值
sampling_defaults:
temperature: 0.6
max_tokens: 4096
top_p: 0.9
frequency_penalty: 0.0
presence_penalty: 0.0
stop:
- "用户:"
- "助手:"
- "###"
- "问题:"
- "回答:"
# 其他配置
dtype: "auto"
quantization: "awq"
port: 2001
api_key: "sk-32b-20240101-ghijk789012"
qwen3_vl_32b_instruct_awq:
path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ"
name: "Qwen3-VL-32B-Instruct-AWQ"
max_model_len: 32768
gpu_memory_utilization: 0.7
port: 2001
dtype: "auto"
quantization: "awq"
tensor_parallel_size: 1
enforce_eager: true
api_key: "sk-glm-20240101-lmnop345678"
# 服务器通用设置
server:
host: "0.0.0.0"
log_level: "info"
# 全局管理员密钥（拥有所有模型的访问权限）
admin_key: "sk-admin-20240101-xyz789"
# 允许的请求头名称（支持多个，按顺序检查）
api_key_headers: ["Authorization", "X-API-Key", "api-key"]
# 是否允许通过查询参数传递密钥
allow_query_param: true
# 查询参数名称
api_key_param: "api_key"
params_config.yaml
+269
View File
@@ -0,0 +1,269 @@
# vLLM 参数配置文件
# 用于定义和识别 vLLM 服务器的各类参数
# 模型参数配置
model_params:
# 必需参数
required:
- path # 模型路径
# 可选参数
optional:
- name # 模型名称
- dtype # 数据类型 (auto, float16, float32, bfloat16)
- quantization # 量化方式 (awq, gptq, squeezellm)
- trust_remote # 是否信任远程代码
# 性能参数配置
performance_params:
# GPU 相关
gpu:
- tensor_parallel_size # 张量并行度
- pipeline_parallel_size # 流水线并行度
- gpu_memory_utilization # GPU 内存利用率 (0.0-1.0)
- swap_space # CPU 交换空间大小 (GB)
- max_num_batched_tokens # 最大批量 tokens 数
- max_num_seqs # 最大并发请求数
- num_scheduler_steps # 调度器步数
# 内存管理
memory:
- block_size # 块大小 (8, 16, 32)
- max_model_len # 最大模型上下文长度
- max_logprobs # 最大 logprobs 数
- disable_sliding_window # 禁用滑动窗口
# 服务参数配置
server_params:
# 网络配置
network:
- host # 服务器主机地址
- port # 服务器端口
- ssl_keyfile # SSL 密钥文件
- ssl_certfile # SSL 证书文件
- ssl_ca_certs # SSL CA 证书
- ssl_keyfile_password # SSL 密钥密码
# API 配置
api:
- api_key # API 密钥
- allowed_origins # 允许的源
- timeout_keep_alive # 保持连接超时时间
# 采样参数配置
sampling_params:
# 温度控制
- temperature # 温度 (0.0-2.0)
- top_p # 核采样参数 (0.0-1.0)
- top_k # Top-K 采样
- min_p # 最小概率
# 惩罚参数
- frequency_penalty # 频率惩罚 (-2.0 到 2.0)
- presence_penalty # 存在惩罚 (-2.0 到 2.0)
- repetition_penalty # 重复惩罚
# 生成长度
- max_tokens # 最大生成 tokens 数
- min_tokens # 最小生成 tokens 数
- stop # 停止词列表
- stop_token_ids # 停止 token IDs
# 其他采样选项
- seed # 随机种子
- use_beam_search # 使用束搜索
- best_of # 束搜索的最佳候选数
- length_penalty # 长度惩罚
- early_stopping # 早期停止
- ignore_eos # 忽略 EOS token
- skip_special_tokens # 跳过特殊 tokens
- spaces_between_special_tokens # 特殊 tokens 之间的空格
# 日志参数配置
logging_params:
- log_level # 日志级别 (debug, info, warning, error)
- log_requests # 是否记录请求
- log_responses # 是否记录响应
# 高级参数配置
advanced_params:
# 执行模式
- enforce_eager # 强制使用 eager 模式
- cuda_graphs # CUDA 图
- use_v2_block_manager # 使用 V2 块管理器
# 注意力后端
- use_rocm_attn # 使用 ROCm 注意力后端
- attention_backend # 注意力后端类型
# 分布式
- distributed_executor_backend # 分布式执行器后端
- ray_workers_use_nsight # Ray workers 使用 nsight
# 其他
- load_format # 加载格式
- download_dir # 下载目录
- revision # 模型版本
- code_revision # 代码版本
- tokenizer_revision # tokenizer 版本
# 参数类型映射
param_types:
# 整数类型
integer:
- tensor_parallel_size
- pipeline_parallel_size
- max_num_batched_tokens
- max_num_seqs
- block_size
- max_model_len
- max_tokens
- min_tokens
- top_k
- best_of
- seed
- num_scheduler_steps
- max_logprobs
- swap_space
- timeout_keep_alive
# 浮点数类型
float:
- gpu_memory_utilization
- temperature
- top_p
- min_p
- frequency_penalty
- presence_penalty
- repetition_penalty
- length_penalty
# 布尔类型
boolean:
- trust_remote
- enforce_eager
- use_rocm_attn
- use_beam_search
- early_stopping
- ignore_eos
- skip_special_tokens
- spaces_between_special_tokens
- log_requests
- log_responses
- disable_sliding_window
- use_v2_block_manager
# 字符串类型
string:
- path
- name
- dtype
- quantization
- host
- port
- api_key
- ssl_keyfile
- ssl_certfile
- ssl_ca_certs
- ssl_keyfile_password
- log_level
- attention_backend
- load_format
- download_dir
- revision
- code_revision
- tokenizer_revision
# 列表类型
list:
- stop
- stop_token_ids
- allowed_origins
# 参数默认值
param_defaults:
host: "0.0.0.0"
port: 8000
dtype: "auto"
gpu_memory_utilization: 0.9
max_model_len: 8192
tensor_parallel_size: 1
max_num_seqs: 256
block_size: 16
temperature: 0.7
top_p: 0.9
max_tokens: 256
enforce_eager: false
trust_remote: false
log_level: "info"
# 参数验证规则
param_validation:
gpu_memory_utilization:
min: 0.0
max: 1.0
temperature:
min: 0.0
max: 2.0
top_p:
min: 0.0
max: 1.0
min_p:
min: 0.0
max: 1.0
frequency_penalty:
min: -2.0
max: 2.0
presence_penalty:
min: -2.0
max: 2.0
tensor_parallel_size:
min: 1
max: 8
block_size:
allowed_values: [8, 16, 32]
# 参数分组（用于配置文件组织）
param_groups:
basic:
name: "基础配置"
params:
- path
- name
- port
- api_key
performance:
name: "性能配置"
params:
- tensor_parallel_size
- gpu_memory_utilization
- max_model_len
- max_num_seqs
- block_size
sampling:
name: "采样配置"
params:
- temperature
- top_p
- max_tokens
- frequency_penalty
- presence_penalty
- stop
advanced:
name: "高级配置"
params:
- dtype
- quantization
- enforce_eager
- trust_remote
- swap_space
scripts/01-rocm-envs.sh
+4
View File
@@ -0,0 +1,4 @@
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
export FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
export VLLM_TARGET_DEVICE=rocm
export VLLM_USE_TRITON_AWQ=1
scripts/99-toolbox-banner.sh
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
# Lightweight banner with machine/GPU and ROCm version (vLLM edition)
# No Triton env sourcing, same info/format as the image/video banner.
# Only show for interactive shells
case $- in *i*) ;; *) return 0 ;; esac
oem_info() {
local v="" m="" d lv lm
for d in /sys/class/dmi/id /sys/devices/virtual/dmi/id; do
[[ -r "$d/sys_vendor" ]] && v=$(<"$d/sys_vendor")
[[ -r "$d/product_name" ]] && m=$(<"$d/product_name")
[[ -n "$v" || -n "$m" ]] && break
done
# ARM/SBC fallback
if [[ -z "$v" && -z "$m" && -r /proc/device-tree/model ]]; then
tr -d '\0' </proc/device-tree/model
return
fi
lv=$(printf '%s' "$v" | tr '[:upper:]' '[:lower:]')
lm=$(printf '%s' "$m" | tr '[:upper:]' '[:lower:]')
if [[ -n "$m" && "$lm" == "$lv "* ]]; then
printf '%s\n' "$m"
else
printf '%s %s\n' "${v:-Unknown}" "${m:-Unknown}"
fi
}
gpu_name() {
local name=""
if command -v rocm-smi >/dev/null 2>&1; then
name=$(rocm-smi --showproductname --csv 2>/dev/null | tail -n1 | cut -d, -f2)
[[ -z "$name" ]] && name=$(rocm-smi --showproductname 2>/dev/null | grep -m1 -E 'Product Name|Card series' | sed 's/.*: //')
fi
if [[ -z "$name" ]] && command -v rocminfo >/dev/null 2>&1; then
name=$(rocminfo 2>/dev/null | awk -F': ' '/^[[:space:]]*Name:/{print $2; exit}')
fi
if [[ -z "$name" ]] && command -v lspci >/dev/null 2>&1; then
name=$(lspci -nn 2>/dev/null | grep -Ei 'vga|display|gpu' | grep -i amd | head -n1 | cut -d: -f3-)
fi
# trim
name=$(printf '%s' "$name" | sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+$//' -e 's/[[:space:]]\{2,\}/ /g')
printf '%s\n' "${name:-Unknown AMD GPU}"
}
rocm_version() {
# Prefer the PyTorch HIP version from the venv, fallback to rocm pkg metadata
local PY="/torch-therock/.venv/bin/python"
[[ -x "$PY" ]] || PY="python"
"$PY" - <<'PY' 2>/dev/null || true
try:
import torch
v = getattr(getattr(torch, "version", None), "hip", "") or ""
if v:
print(v)
else:
raise Exception("no torch.version.hip")
except Exception:
try:
import importlib.metadata as im
try:
print(im.version("_rocm_sdk_core"))
except Exception:
print(im.version("rocm"))
except Exception:
print("")
PY
}
MACHINE="$(oem_info)"
GPU="$(gpu_name)"
ROCM_VER="$(rocm_version)"
echo
cat <<'ASCII'
_____ _____ ______ ____ _ _ ___ ______ ___ ___
| __ \ /\ | __ \| ____/ __ \| \ | | / _ \____ / _ \ / _ \
| |__) | / \ | | | | |__ | | | | \| | | (_) | / / | | | | | |
| _ / / /\ \ | | | | __|| | | | . ` | \__, | / /| | | | | | |
| | \ \ / ____ \| |__| | |___| |__| | |\ | / / / / | |_| | |_| |
|_| \_\/_/ \_\_____/|______\____/|_| \_| /_/ /_/ \___/ \___/
_____ _____ _____ ____
/\ |_ _| | __ \| __ \ / __ \
/ \ | | | |__) | |__) | | | |
/ /\ \ | | | ___/| _ /| | | |
/ ____ \ _| |_ | | | | \ \| |__| |
/_/ \_\_____| |_| |_| \_\\____/
v L L M
ASCII
echo
printf 'AMD R9700 — vLLM Toolbox (gfx1201, ROCm via TheRock)\n'
[[ -n "$ROCM_VER" ]] && printf 'ROCm nightly: %s\n' "$ROCM_VER"
echo
printf 'Machine: %s\n' "$MACHINE"
printf 'GPU : %s\n\n' "$GPU"
printf 'Repo : https://github.com/kyuz0/amd-r9700-vllm-toolboxes\n'
printf 'Image : docker.io/kyuz0/vllm-therock-gfx1201:latest\n\n'
printf 'Included:\n'
printf ' - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling"
printf ' - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct"
printf ' - %-16s → %s\n' "API test" "curl localhost:8000/v1/chat/completions"
echo
printf 'SSH tip: ssh -L 8000:localhost:8000 user@host\n\n'
unset PROMPT_COMMAND
PS1='\u@\h:\w\$ '
scripts/start_vllm.py
+235
View File
@@ -0,0 +1,235 @@
#!/usr/bin/env python3
import sys
import os
import json
import yaml
import shutil
import subprocess
from pathlib import Path
# Add benchmarks dir to path to import config
SCRIPT_DIR = Path(__file__).parent.resolve()
OPT_DIR = Path("/opt")
# Required environment variable pointing to a local models directory
LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
if not LOCAL_MODEL_DIR:
print("Error: LOCAL_MODEL_DIR environment variable is required.")
sys.exit(1)
# Configuration file path
CONFIG_FILE = os.getenv("VLLM_CONFIG_FILE", "/etc/vllm/model_config.yaml")
# Default configuration
DEFAULT_CONFIG = {
"default": "",
"models": {},
"server": {
"host": "0.0.0.0",
"log_level": "info"
}
}
def detect_gpus():
"""Detects AMD GPUs via rocm-smi or /dev/dri."""
try:
# Try rocm-smi first
res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
if res.returncode == 0:
count = res.stdout.count("GPU")
if count > 0: return count
except: pass
# Fallback to /dev/dri/render*
try:
return len(list(Path("/dev/dri").glob("renderD*")))
except:
return 1
def load_config():
"""Load configuration from YAML file."""
config = DEFAULT_CONFIG.copy()
if Path(CONFIG_FILE).exists():
try:
with open(CONFIG_FILE, "r", encoding="utf-8") as f:
user_config = yaml.safe_load(f)
if user_config:
config.update(user_config)
except Exception as e:
print(f"Warning: Failed to load config file: {e}")
print("Using default configuration.")
else:
print(f"Warning: Config file not found at {CONFIG_FILE}")
print("Using default configuration.")
return config
def nuke_vllm_cache():
"""Removes vLLM cache directory to fix potential graph/incompatibility issues."""
cache = Path.home() / ".cache" / "vllm"
if cache.exists():
try:
print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
subprocess.run(["rm", "-rf", str(cache)], check=True)
cache.mkdir(parents=True, exist_ok=True)
print(" Done.")
except Exception as e:
print(f" Failed: {e}")
def get_local_model_path(model_name):
"""Get local model path from LOCAL_MODEL_DIR."""
# Try multiple paths:
# 1) LOCAL_MODEL_DIR/model_name
# 2) case-insensitive match in LOCAL_MODEL_DIR
# Exact match
candidate_exact = os.path.join(LOCAL_MODEL_DIR, model_name)
if os.path.isdir(candidate_exact):
return candidate_exact
# Case-insensitive match
try:
for entry in os.listdir(LOCAL_MODEL_DIR):
if entry.lower() == model_name.lower():
entry_path = os.path.join(LOCAL_MODEL_DIR, entry)
if os.path.isdir(entry_path):
return entry_path
except Exception as e:
print(f"Error searching for model: {e}")
return None
def verify_model_path(model_path):
"""Verify that the model path contains required files."""
required = ["config.json", "pytorch_model.bin", "model.safetensors"]
found = any(os.path.isfile(os.path.join(model_path, f)) for f in required)
if not found:
print(f"Error: local model dir {model_path} missing expected files {required}")
return False
return True
def main():
# Load configuration
config = load_config()
# Get model name from command line or use default
model_name = None
if len(sys.argv) > 1:
model_name = sys.argv[1]
if not model_name:
model_name = config.get("default", "")
if not model_name:
print("Error: Either specify a model name as argument or set 'default' in config.")
sys.exit(1)
# Get model configuration
model_config = config.get("models", {}).get(model_name)
if not model_config:
print(f"Error: Model '{model_name}' not found in configuration.")
sys.exit(1)
# Get model path
model_path = model_config.get("path")
if not model_path:
# Fallback to LOCAL_MODEL_DIR if path not specified
model_path = get_local_model_path(model_name)
if not model_path:
print(f"Error: model '{model_name}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}")
sys.exit(1)
# Verify model path
if not verify_model_path(model_path):
sys.exit(1)
# Detect GPU count
gpu_count = detect_gpus()
# Get server configuration
server_config = config.get("server", {})
host = server_config.get("host", "0.0.0.0")
# Build command
cmd = ["vllm", "serve", model_path]
# Add server parameters
cmd.extend(["--host", host])
if "port" in model_config:
cmd.extend(["--port", str(model_config["port"])])
# Add API key for OpenAI compatibility
api_key = model_config.get("api_key")
if api_key:
cmd.extend(["--api-key", api_key])
# Add model parameters
if "tensor_parallel_size" in model_config:
tp_size = min(model_config["tensor_parallel_size"], gpu_count)
cmd.extend(["--tensor-parallel-size", str(tp_size)])
if "max_num_seqs" in model_config:
cmd.extend(["--max-num-seqs", str(model_config["max_num_seqs"])])
if "max_model_len" in model_config:
cmd.extend(["--max-model-len", str(model_config["max_model_len"])])
if "gpu_memory_utilization" in model_config:
cmd.extend(["--gpu-memory-utilization", str(model_config["gpu_memory_utilization"])])
if "dtype" in model_config:
cmd.extend(["--dtype", model_config["dtype"]])
else:
cmd.extend(["--dtype", "auto"])
if "max_num_batched_tokens" in model_config:
cmd.extend(["--max-num-batched-tokens", str(model_config["max_num_batched_tokens"])])
if "block_size" in model_config:
cmd.extend(["--block-size", str(model_config["block_size"])])
if "swap_space" in model_config:
cmd.extend(["--swap-space", str(model_config["swap_space"])])
if model_config.get("enforce_eager", False):
cmd.append("--enforce-eager")
if model_config.get("trust_remote", False):
cmd.append("--trust-remote-code")
# Set environment variables
env = os.environ.copy()
if model_config.get("use_rocm_attn", False):
env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
# Clear cache if requested
if model_config.get("clear_cache", False):
nuke_vllm_cache()
# Print configuration
print("\n" + "="*60)
print(f" Launching: {model_name}")
print(f" Model Path: {model_path}")
print(f" Host: {host}")
if "port" in model_config:
print(f" Port: {model_config['port']}")
if api_key:
print(f" API Key: {api_key[:8]}...{api_key[-4:]}")
if "tensor_parallel_size" in model_config:
print(f" TP Size: {tp_size}")
if "max_num_seqs" in model_config:
print(f" Max Seqs: {model_config['max_num_seqs']}")
if "max_model_len" in model_config:
print(f" Max Ctx: {model_config['max_model_len']}")
if "gpu_memory_utilization" in model_config:
print(f" GPU Util: {model_config['gpu_memory_utilization']}")
print(f" Command: {' '.join(cmd)}")
print("="*60 + "\n")
# Launch vLLM server
os.execvpe("vllm", cmd, env)
if __name__ == "__main__":
main()
scripts/zz-venv-last.sh
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Ensure /opt/venv/bin is first even if ~/.local/bin or ~/.cargo/bin prepend themselves via user dotfiles.
_venv_path_fix() {
# remove any existing /opt/venv/bin entries, then prepend one
local newpath
newpath="$(printf '%s' "$PATH" | awk -v RS=: -v ORS=: '$0!="/opt/venv/bin"{print}')"
PATH="/opt/venv/bin:${newpath%:}"
}
# run once after shell init; don't duplicate
case "$PROMPT_COMMAND" in
*_venv_path_fix*) : ;;
*) PROMPT_COMMAND="_venv_path_fix${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;;
esac