diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..10b731c --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,5 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml +# 基于编辑器的 HTTP 客户端请求 +/httpRequests/ diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..e2a5931 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..6a9e3aa --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..504e620 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vllm-r9700-container.iml b/.idea/vllm-r9700-container.iml new file mode 100644 index 0000000..2a15c21 --- /dev/null +++ b/.idea/vllm-r9700-container.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1d94a13 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,172 @@ +FROM registry.fedoraproject.org/fedora:43 + +# 1. System Base & Build Tools +RUN dnf -y install --setopt=install_weak_deps=False --nodocs \ + python3.13 python3.13-devel git rsync libatomic bash ca-certificates curl \ + gcc gcc-c++ binutils make ffmpeg-free \ + cmake ninja-build aria2c tar xz vim nano \ + libdrm-devel zlib-devel openssl-devel jq \ + numactl-devel gperftools-libs procps-ng \ + && dnf clean all && rm -rf /var/cache/dnf/* + +# 2. Install "TheRock" ROCm SDK (Tarball Method) +WORKDIR /tmp +ARG ROCM_MAJOR_VER=7 +ARG GFX=gfx120X-all +RUN set -euo pipefail; \ + BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \ + PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \ + KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \ + | tr '<' '\n' \ + | grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \ + | sort -V | tail -n1)"; \ + echo "Downloading Latest Tarball: ${KEY}"; \ + aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz; \ + mkdir -p /opt/rocm; \ + tar xzf therock.tar.gz -C /opt/rocm --strip-components=1; \ + rm therock.tar.gz + +# 3. Configure Global ROCm Environment +RUN export ROCM_PATH=/opt/rocm && \ + BITCODE_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \ + printf '%s\n' \ + "export ROCM_PATH=/opt/rocm" \ + "export HIP_PLATFORM=amd" \ + "export HIP_PATH=/opt/rocm" \ + "export HIP_CLANG_PATH=/opt/rocm/llvm/bin" \ + "export HIP_DEVICE_LIB_PATH=$BITCODE_PATH" \ + "export PATH=$ROCM_PATH/bin:$ROCM_PATH/llvm/bin:\$PATH" \ + "export LD_LIBRARY_PATH=$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib:\$LD_LIBRARY_PATH" \ + "export ROCBLAS_USE_HIPBLASLT=1" \ + "export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" \ + "export VLLM_TARGET_DEVICE=rocm" \ + "export HIP_FORCE_DEV_KERNARG=1" \ + "export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1" \ + "export LD_PRELOAD=/usr/lib64/libtcmalloc_minimal.so.4" \ + > /etc/profile.d/rocm-sdk.sh && \ + chmod 0644 /etc/profile.d/rocm-sdk.sh + +# 4. Python Venv Setup +RUN /usr/bin/python3.13 -m venv /opt/venv +ENV VIRTUAL_ENV=/opt/venv +ENV PATH=/opt/venv/bin:$PATH +ENV PIP_NO_CACHE_DIR=1 +RUN printf 'source /opt/venv/bin/activate\n' > /etc/profile.d/venv.sh +RUN python -m pip install --upgrade pip wheel packaging "setuptools<80.0.0" + +# 5. Install PyTorch (TheRock Nightly) and PyYAML +RUN python -m pip install \ + --index-url https://rocm.nightlies.amd.com/v2-staging/gfx120X-all/ \ + --pre torch torchaudio torchvision && \ + python -m pip install pyyaml + +# Flash-Attention +WORKDIR /opt +ENV FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE" + +RUN git clone https://github.com/ROCm/flash-attention.git &&\ \ + cd flash-attention &&\ + git checkout main_perf &&\ + python setup.py install && \ + cd /opt && rm -rf /opt/flash-attention + +# 6. Clone vLLM +RUN git clone https://github.com/vllm-project/vllm.git /opt/vllm +WORKDIR /opt/vllm + +# --- PATCHING --- +RUN echo "import sys, re" > patch_vllm.py && \ + echo "from pathlib import Path" >> patch_vllm.py && \ + echo "p = Path('vllm/platforms/__init__.py')" >> patch_vllm.py && \ + echo "txt = p.read_text()" >> patch_vllm.py && \ + echo "txt = txt.replace('import amdsmi', '# import amdsmi')" >> patch_vllm.py && \ + echo "txt = re.sub(r'is_rocm = .*', 'is_rocm = True', txt)" >> patch_vllm.py && \ + echo "txt = re.sub(r'if len\(amdsmi\.amdsmi_get_processor_handles\(\)\) > 0:', 'if True:', txt)" >> patch_vllm.py && \ + echo "txt = txt.replace('amdsmi.amdsmi_init()', 'pass')" >> patch_vllm.py && \ + echo "txt = txt.replace('amdsmi.amdsmi_shut_down()', 'pass')" >> patch_vllm.py && \ + echo "p.write_text(txt)" >> patch_vllm.py && \ + echo "p = Path('vllm/platforms/rocm.py')" >> patch_vllm.py && \ + echo "txt = p.read_text()" >> patch_vllm.py && \ + echo "header = 'import sys\nfrom unittest.mock import MagicMock\nsys.modules[\"amdsmi\"] = MagicMock()\n'" >> patch_vllm.py && \ + echo "txt = header + txt" >> patch_vllm.py && \ + echo "txt = re.sub(r'device_type = .*', 'device_type = \"rocm\"', txt)" >> patch_vllm.py && \ + echo "txt = re.sub(r'device_name = .*', 'device_name = \"gfx1201\"', txt)" >> patch_vllm.py && \ + echo "txt += '\n def get_device_name(self, device_id: int = 0) -> str:\n return \"AMD-gfx1201\"\n'" >> patch_vllm.py && \ + echo "p.write_text(txt)" >> patch_vllm.py && \ + echo "print('Successfully patched vLLM for R9700')" >> patch_vllm.py && \ + python patch_vllm.py + +# 7. Build vLLM (Wheel Method) with CLANG Host Compiler +RUN python -m pip install --upgrade cmake ninja packaging wheel numpy "setuptools-scm>=8" "setuptools<80.0.0" scikit-build-core pybind11 +ENV ROCM_HOME="/opt/rocm" +ENV HIP_PATH="/opt/rocm" +ENV VLLM_TARGET_DEVICE="rocm" +ENV PYTORCH_ROCM_ARCH="gfx1201" +ENV HIP_ARCHITECTURES="gfx1201" +ENV AMDGPU_TARGETS="gfx1201" +ENV MAX_JOBS="4" + +# --- FIX FOR SEGFAULT --- +ENV CC="/opt/rocm/llvm/bin/clang" +ENV CXX="/opt/rocm/llvm/bin/clang++" + +RUN export HIP_DEVICE_LIB_PATH=$(find /opt/rocm -type d -name bitcode -print -quit) && \ + echo "Compiling with Bitcode: $HIP_DEVICE_LIB_PATH" && \ + export CMAKE_PREFIX_PATH="/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \ + export CMAKE_ARGS="-DROCM_PATH=/opt/rocm -DHIP_PATH=/opt/rocm -DAMDGPU_TARGETS=gfx1201 -DHIP_ARCHITECTURES=gfx1201 -DCMAKE_PREFIX_PATH=/opt/venv/lib64/python3.13/site-packages/torch/share/cmake:/opt/rocm" && \ + python -m pip wheel --no-build-isolation --no-deps -w /tmp/dist -v . && \ + python -m pip install /tmp/dist/*.whl + +# --- bitsandbytes (ROCm) --- +WORKDIR /opt +RUN git clone -b rocm_enabled_multi_backend https://github.com/ROCm/bitsandbytes.git +WORKDIR /opt/bitsandbytes + +# Explicitly set HIP_PLATFORM (Docker ENV, not /etc/profile) +ENV HIP_PLATFORM="amd" +ENV CMAKE_PREFIX_PATH="/opt/rocm" + +# Force CMake to use the System ROCm Compiler (/opt/rocm/llvm/bin/clang++) +RUN cmake -S . \ + -DGPU_TARGETS="gfx1201" \ + -DBNB_ROCM_ARCH="gfx1201" \ + -DCOMPUTE_BACKEND=hip \ + -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \ + -DCMAKE_CXX_COMPILER=/opt/rocm/llvm/bin/clang++ \ + && \ + make -j$(nproc) && \ + python -m pip install --no-cache-dir . --no-build-isolation --no-deps + +# 8. Final Cleanup & Runtime +WORKDIR /opt +RUN chmod -R a+rwX /opt && \ + find /opt/venv -type f -name "*.so" -exec strip -s {} + 2>/dev/null || true && \ + find /opt/venv -type d -name "__pycache__" -prune -exec rm -rf {} + && \ + rm -rf /root/.cache/pip || true && \ + dnf clean all && rm -rf /var/cache/dnf/* + +# Create vLLM configuration directory +RUN mkdir -p /etc/vllm + +# Create default YAML configuration file +RUN printf '# model_config.yaml - 多模型配置\n# 默认启动的模型\ndefault: "deepseek_r1_distill_qwen_32b_awq"\n\n# 模型配置\nmodels:\n deepseek_r1_distill_qwen_14b:\n path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B"\n name: "DeepSeek-R1-Distill-Qwen-14B"\n max_model_len: 8192\n gpu_memory_utilization: 0.9\n port: 2001\n dtype: "float16"\n quantization: "awq"\n tensor_parallel_size: 1\n enforce_eager: true\n api_key: "sk-14b-20240101-abcdef123456"\n \n deepseek_r1_distill_qwen_32b_awq:\n path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ"\n name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"\n \n # 模型性能参数\n max_model_len: 32768\n gpu_memory_utilization: 0.95\n enforce_eager: true\n max_num_seqs: 2\n max_num_batched_tokens: 1024\n block_size: 16\n tensor_parallel_size: 1\n swap_space: 0\n \n # 新增：采样参数默认值\n sampling_defaults:\n temperature: 0.6\n max_tokens: 4096\n top_p: 0.9\n frequency_penalty: 0.0\n presence_penalty: 0.0\n stop:\n - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + \n # 其他配置\n dtype: "auto"\n quantization: "awq"\n port: 2001\n api_key: "sk-32b-20240101-ghijk789012"\n \n glm_4_7_flash_awq:\n path: "/app/vllm/models/GLM-4.7-Flash-AWQ"\n name: "GLM-4.7-Flash-AWQ"\n \n # 模型性能参数\n max_model_len: 32768\n gpu_memory_utilization: 0.9\n enforce_eager: false\n max_num_seqs: 2\n max_num_batched_tokens: 1024\n block_size: 16\n tensor_parallel_size: 1\n swap_space: 0\n \n # 新增：采样参数默认值\n sampling_defaults:\n temperature: 0.6\n max_tokens: 4096\n top_p: 0.9\n frequency_penalty: 0.0\n presence_penalty: 0.0\n stop:\n - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + \n # 其他配置\n dtype: "auto"\n quantization: "awq"\n port: 2001\n api_key: "sk-32b-20240101-ghijk789012"\n \n qwen3_vl_32b_instruct_awq:\n path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ"\n name: "Qwen3-VL-32B-Instruct-AWQ"\n max_model_len: 32768\n gpu_memory_utilization: 0.7\n port: 2001\n dtype: "auto"\n quantization: "awq"\n tensor_parallel_size: 1\n enforce_eager: true\n api_key: "sk-glm-20240101-lmnop345678"\n\n# 服务器通用设置\nserver:\n host: "0.0.0.0"\n log_level: "info"\n # 全局管理员密钥（拥有所有模型的访问权限）\n admin_key: "sk-admin-20240101-xyz789"\n # 允许的请求头名称（支持多个，按顺序检查）\n api_key_headers: ["Authorization", "X-API-Key", "api-key"]\n # 是否允许通过查询参数传递密钥\n allow_query_param: true\n # 查询参数名称\n api_key_param: "api_key"' > /etc/vllm/model_config.yaml + +# Copy necessary scripts +COPY scripts/01-rocm-envs.sh /etc/profile.d/01-rocm-envs.sh +COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh +COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh +COPY scripts/start_vllm.py /usr/local/bin/start-vllm +RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm +RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh + +CMD ["/bin/bash"] \ No newline at end of file diff --git a/README.md b/README.md index a7892c6..c8c0f71 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,444 @@ -# vllm-r9700-container +# 自定义 vLLM 容器 +这是一个基于 Fedora 的 Docker/Podman 容器，专为在 AMD Radeon R9700 (gfx1201) GPU 上运行 vLLM 而设计。 + +## 特性 + +- 基于 Fedora 43 +- 使用最新的 TheRock ROCm 7.x SDK +- 包含 PyTorch 预发布版本（ROCm 支持） +- 内置 Flash-Attention（ROCm 版本） +- 支持多种大型语言模型 +- 基于配置文件的启动方式 +- 仅支持本地模型（无网络下载功能） +- 多模型配置支持 +- **提供标准的 OpenAI API 服务** + +## 快速开始 + +### 1. 构建并运行（推荐） + +```bash +# 赋予脚本执行权限 +chmod +x build_and_run.sh + +# 一键构建并运行 +./build_and_run.sh +``` + +### 2. 手动构建和运行 + +```bash +# 构建镜像 +docker build -t custom-vllm-r9700:latest . + +# 运行容器 +docker run -it --device /dev/dri --device /dev/kfd \ + --group-add video --group-add render --security-opt seccomp=unconfined \ + -v /path/to/models:/models \ + -v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \ + -e LOCAL_MODEL_DIR=/models \ + custom-vllm-r9700:latest +``` + +## 构建容器 + +在项目目录中运行： + +```bash +docker build -t custom-vllm-r9700:latest . +``` + +## 使用方法 + +### 使用 Docker/Podman + +```bash +docker run -it --device /dev/dri --device /dev/kfd \ + --group-add video --group-add render --security-opt seccomp=unconfined \ + -v /path/to/models:/models \ + -v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \ + -e LOCAL_MODEL_DIR=/models \ + custom-vllm-r9700:latest +``` + +### 使用 Toolbx（Fedora） + +```bash +toolbox create vllm-custom \ + --image custom-vllm-r9700:latest \ + -- --device /dev/dri --device /dev/kfd \ + --group-add video --group-add render --security-opt seccomp=unconfined + +toolbox enter vllm-custom +``` + +### 使用 Distrobox（Ubuntu） + +```bash +distrobox create -n vllm-custom \ + --image custom-vllm-r9700:latest \ + --additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined" + +distrobox enter vllm-custom +``` + +## 配置文件 + +容器使用 YAML 格式的配置文件来设置 vLLM 服务器参数。默认配置文件位于 `/etc/vllm/model_config.yaml`。项目根目录中提供了配置文件示例 `model_config.yaml.example`，您可以参考它来创建自己的配置文件。 + +### 配置文件示例 + +```yaml +# model_config.yaml - 多模型配置示例 +# 默认启动的模型 +default: "deepseek_r1_distill_qwen_32b_awq" + +# 模型配置 +models: + deepseek_r1_distill_qwen_14b: + path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B" + name: "DeepSeek-R1-Distill-Qwen-14B" + max_model_len: 8192 + gpu_memory_utilization: 0.9 + port: 2001 + dtype: "float16" + quantization: "awq" + tensor_parallel_size: 1 + enforce_eager: true + api_key: "sk-14b-20240101-abcdef123456" + + deepseek_r1_distill_qwen_32b_awq: + path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ" + name: "DeepSeek-R1-Distill-Qwen-32B-AWQ" + + # 模型性能参数 + max_model_len: 32768 + gpu_memory_utilization: 0.95 + enforce_eager: true + max_num_seqs: 2 + max_num_batched_tokens: 1024 + block_size: 16 + tensor_parallel_size: 1 + swap_space: 0 + + # 新增：采样参数默认值 + sampling_defaults: + temperature: 0.6 + max_tokens: 4096 + top_p: 0.9 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: + - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + + # 其他配置 + dtype: "auto" + quantization: "awq" + port: 2001 + api_key: "sk-32b-20240101-ghijk789012" + + glm_4_7_flash_awq: + path: "/app/vllm/models/GLM-4.7-Flash-AWQ" + name: "GLM-4.7-Flash-AWQ" + + # 模型性能参数 + max_model_len: 32768 + gpu_memory_utilization: 0.9 + enforce_eager: false + max_num_seqs: 2 + max_num_batched_tokens: 1024 + block_size: 16 + tensor_parallel_size: 1 + swap_space: 0 + + # 新增：采样参数默认值 + sampling_defaults: + temperature: 0.6 + max_tokens: 4096 + top_p: 0.9 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: + - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + + # 其他配置 + dtype: "auto" + quantization: "awq" + port: 2001 + api_key: "sk-32b-20240101-ghijk789012" + + qwen3_vl_32b_instruct_awq: + path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ" + name: "Qwen3-VL-32B-Instruct-AWQ" + max_model_len: 32768 + gpu_memory_utilization: 0.7 + port: 2001 + dtype: "auto" + quantization: "awq" + tensor_parallel_size: 1 + enforce_eager: true + api_key: "sk-glm-20240101-lmnop345678" + +# 服务器通用设置 +server: + host: "0.0.0.0" + log_level: "info" + # 全局管理员密钥（拥有所有模型的访问权限） + admin_key: "sk-admin-20240101-xyz789" + # 允许的请求头名称（支持多个，按顺序检查） + api_key_headers: ["Authorization", "X-API-Key", "api-key"] + # 是否允许通过查询参数传递密钥 + allow_query_param: true + # 查询参数名称 + api_key_param: "api_key" +``` + +### 配置参数说明 + +- `default`：默认启动的模型名称 +- `models`：模型配置列表 + - 每个模型包含： + - `path`：模型路径 + - `name`：模型名称 + - `max_model_len`：最大模型上下文长度 + - `gpu_memory_utilization`：GPU 内存利用率 + - `port`：服务器端口 + - `dtype`：数据类型 + - `quantization`：量化方式 + - `tensor_parallel_size`：张量并行度 + - `enforce_eager`：是否强制使用 eager 模式 + - `max_num_seqs`：最大并发请求数 + - `max_num_batched_tokens`：最大批量 tokens 数 + - `block_size`：块大小 + - `swap_space`：交换空间大小 + - `sampling_defaults`：采样参数默认值 + - `api_key`：API 密钥（用于 OpenAI 兼容模式） +- `server`：服务器通用设置 + - `host`：服务器主机地址 + - `log_level`：日志级别 + - `admin_key`：全局管理员密钥 + - `api_key_headers`：允许的请求头名称 + - `allow_query_param`：是否允许通过查询参数传递密钥 + - `api_key_param`：查询参数名称 + +## 环境变量 + +- `LOCAL_MODEL_DIR`：本地模型目录路径（必须设置） +- `VLLM_CONFIG_FILE`：配置文件路径（默认：/etc/vllm/model_config.yaml） + +## 启动 vLLM 服务器 + +### 使用构建和部署脚本（推荐） + +项目提供了 `build_and_run.sh` 脚本，用于一键构建镜像并部署运行： + +```bash +# 赋予执行权限 +chmod +x build_and_run.sh + +# 构建并运行（默认后台运行） +./build_and_run.sh + +# 仅构建镜像 +./build_and_run.sh -b + +# 重新构建镜像（不使用缓存） +./build_and_run.sh -r + +# 仅停止并删除容器 +./build_and_run.sh -s + +# 指定端口运行 +./build_and_run.sh -p 8080 + +# 指定配置文件和模型目录 +./build_and_run.sh -c /path/to/config.yaml -m /path/to/models + +# 交互式运行（前台运行） +./build_and_run.sh -i +``` + +**脚本选项：** + +- `-b, --build-only`：仅构建镜像，不运行容器 +- `-r, --rebuild`：重新构建镜像（不使用缓存） +- `-s, --stop`：仅停止并删除容器 +- `-d, --detach`：后台运行容器（默认） +- `-i, --interactive`：交互式运行容器 +- `-c, --config FILE`：指定配置文件路径 +- `-m, --models DIR`：指定模型目录路径 +- `-p, --port PORT`：指定服务端口 +- `-h, --help`：显示帮助信息 + +### 手动部署 + +```bash +# 1. 构建镜像 +docker build -t custom-vllm-r9700:latest . + +# 2. 运行容器 +docker run -it --device /dev/dri --device /dev/kfd \ + --group-add video --group-add render --security-opt seccomp=unconfined \ + -v /path/to/models:/models \ + -v /path/to/model_config.yaml:/etc/vllm/model_config.yaml \ + -e LOCAL_MODEL_DIR=/models \ + custom-vllm-r9700:latest +``` + +### 容器内运行 + +进入容器后，可以使用以下命令启动 vLLM 服务器： + +```bash +# 使用默认模型启动 +start-vllm + +# 或指定模型名称启动 +start-vllm deepseek_r1_distill_qwen_14b + +# 或直接指定模型路径 +vllm serve /models/model-name --tensor-parallel-size 2 --max-model-len 128000 +``` + +## 测试 API（OpenAI 兼容） + +vLLM 提供与 OpenAI API 完全兼容的服务接口。 + +### 1. 使用 curl 测试 + +```bash +# 聊天补全接口 +curl -X POST http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-32b-20240101-ghijk789012" \ + -d '{ + "model": "deepseek_r1_distill_qwen_32b_awq", + "messages": [ + {"role": "user", "content": "你好，请介绍一下你自己"} + ], + "temperature": 0.6, + "max_tokens": 4096, + "top_p": 0.9 + }' + +# 文本补全接口 +curl -X POST http://localhost:8000/v1/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-32b-20240101-ghijk789012" \ + -d '{ + "model": "deepseek_r1_distill_qwen_32b_awq", + "prompt": "Once upon a time", + "max_tokens": 100 + }' + +# 列出可用模型 +curl http://localhost:8000/v1/models \ + -H "Authorization: Bearer sk-32b-20240101-ghijk789012" +``` + +### 2. 使用 Python OpenAI SDK + +```python +from openai import OpenAI + +# 初始化客户端 +client = OpenAI( + base_url="http://localhost:8000/v1", + api_key="sk-32b-20240101-ghijk789012" +) + +# 聊天补全 +response = client.chat.completions.create( + model="deepseek_r1_distill_qwen_32b_awq", + messages=[ + {"role": "user", "content": "你好，请介绍一下你自己"} + ], + temperature=0.6, + max_tokens=4096 +) + +print(response.choices[0].message.content) + +# 文本补全 +response = client.completions.create( + model="deepseek_r1_distill_qwen_32b_awq", + prompt="Once upon a time", + max_tokens=100 +) + +print(response.choices[0].text) +``` + +### 3. 使用其他 OpenAI 兼容工具 + +由于提供标准的 OpenAI API，您可以使用任何支持 OpenAI 的工具和库，例如： + +- LangChain +- LlamaIndex +- AutoGen +- FastChat +- 等等 + +只需将 `base_url` 设置为 `http://localhost:8000/v1`，并使用配置的 API 密钥即可。 + +## 本地模型目录结构 + +确保本地模型目录包含以下文件之一： +- `config.json` +- `pytorch_model.bin` +- `model.safetensors` + +正确的目录结构示例： + +``` +/models/ + ├── deepseek_r1_distill_qwen_14b/ + │ ├── config.json + │ └── model.safetensors + ├── deepseek_r1_distill_qwen_32b_awq/ + │ ├── config.json + │ └── pytorch_model.bin + ├── glm_4_7_flash_awq/ + │ ├── config.json + │ └── model.safetensors + └── qwen3_vl_32b_instruct_awq/ + ├── config.json + └── pytorch_model.bin +``` + +## 注意事项 + +- 确保您的 AMD Radeon R9700 GPU 驱动已正确安装 +- 容器需要访问 GPU 设备，因此运行时需要添加 `--device /dev/dri --device /dev/kfd` 参数 +- 首次启动时，vLLM 会编译计算图，可能需要较长时间 +- 如果遇到内存不足的问题，可以调整 `gpu_memory_utilization` 参数 +- API 密钥在配置文件的每个模型中单独配置，用于 OpenAI 兼容模式的认证 + +## OpenAI API 兼容性 + +本容器提供的服务完全兼容 OpenAI API 标准，包括： + +- **聊天补全**：`/v1/chat/completions` +- **文本补全**：`/v1/completions` +- **模型列表**：`/v1/models` +- **嵌入**：`/v1/embeddings`（如支持） + +所有端点都支持标准的 OpenAI 请求格式和参数，您可以无缝切换使用。 + +## 项目文件说明 + +- `Dockerfile` - Docker 镜像构建文件 +- `README.md` - 项目说明文档 +- `model_config.yaml.example` - 配置文件示例 +- `params_config.yaml` - 参数配置文件（定义各类参数的类型和验证规则） +- `build_and_run.sh` - 构建和部署脚本 +- `scripts/start_vllm.py` - vLLM 启动脚本 diff --git a/build_and_run.sh b/build_and_run.sh new file mode 100644 index 0000000..cd205cf --- /dev/null +++ b/build_and_run.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +# vLLM R9700 Container 构建和部署脚本 +# 用于一键构建镜像并部署运行 + +set -e + +# 配置变量 +IMAGE_NAME="custom-vllm-r9700" +IMAGE_TAG="latest" +CONTAINER_NAME="custom-vllm-r9700" +MODEL_DIR="${MODEL_DIR:-/opt/models}" +CONFIG_FILE="${CONFIG_FILE:-/opt/model_config.yaml}" +LOCAL_MODEL_DIR="${LOCAL_MODEL_DIR:-/models}" + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# 打印带颜色的消息 +print_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# 显示使用帮助 +show_usage() { + echo "用法：$0 [选项]" + echo "" + echo "选项:" + echo " -b, --build-only 仅构建镜像，不运行容器" + echo " -r, --rebuild 重新构建镜像（不使用缓存）" + echo " -s, --stop 仅停止并删除容器" + echo " -d, --detach 后台运行容器（默认）" + echo " -i, --interactive 交互式运行容器" + echo " -c, --config FILE 指定配置文件路径" + echo " -m, --models DIR 指定模型目录路径" + echo " -p, --port PORT 指定服务端口" + echo " -h, --help 显示此帮助信息" + echo "" + echo "示例:" + echo " $0 # 构建并运行" + echo " $0 -b # 仅构建镜像" + echo " $0 -r # 重新构建并运行" + echo " $0 -s # 停止并删除容器" + echo " $0 -p 8080 # 使用端口 8080 运行" + echo " $0 -c /path/to/config.yaml # 使用指定配置文件" + echo " $0 -m /path/to/models # 使用指定模型目录" + exit 0 +} + +# 解析命令行参数 +BUILD_ONLY=false +REBUILD=false +STOP_ONLY=false +DETACH=true +CUSTOM_CONFIG="" +CUSTOM_MODELS="" +CUSTOM_PORT="" + +while [[ $# -gt 0 ]]; do + case $1 in + -b|--build-only) + BUILD_ONLY=true + shift + ;; + -r|--rebuild) + REBUILD=true + shift + ;; + -s|--stop) + STOP_ONLY=true + shift + ;; + -d|--detach) + DETACH=true + shift + ;; + -i|--interactive) + DETACH=false + shift + ;; + -c|--config) + CUSTOM_CONFIG="$2" + shift 2 + ;; + -m|--models) + CUSTOM_MODELS="$2" + shift 2 + ;; + -p|--port) + CUSTOM_PORT="$2" + shift 2 + ;; + -h|--help) + show_usage + ;; + *) + print_error "未知选项：$1" + show_usage + ;; + esac +done + +# 停止并删除容器 +stop_container() { + print_info "停止并删除容器..." + + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + docker stop "${CONTAINER_NAME}" 2>/dev/null || true + print_success "容器已停止" + + docker rm "${CONTAINER_NAME}" 2>/dev/null || true + print_success "容器已删除" + else + print_warning "容器不存在" + fi +} + +# 如果只需要停止容器，直接退出 +if [ "$STOP_ONLY" = true ]; then + stop_container + exit 0 +fi + +# 检查 Docker 是否运行 +check_docker() { + if ! docker info > /dev/null 2>&1; then + print_error "Docker 未运行或无权限访问" + print_error "请确保 Docker 服务正在运行，并且当前用户有 Docker 访问权限" + exit 1 + fi + print_success "Docker 检查通过" +} + +# 构建镜像 +build_image() { + print_info "开始构建 Docker 镜像..." + print_info "镜像名称：${IMAGE_NAME}:${IMAGE_TAG}" + + BUILD_ARGS="--no-cache" + if [ "$REBUILD" = false ]; then + BUILD_ARGS="" + fi + + if docker build ${BUILD_ARGS} -t "${IMAGE_NAME}:${IMAGE_TAG}" .; then + print_success "镜像构建成功" + docker images "${IMAGE_NAME}:${IMAGE_TAG}" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}" + else + print_error "镜像构建失败" + exit 1 + fi +} + +# 运行容器 +run_container() { + print_info "开始部署容器..." + + # 设置卷挂载参数 + VOLUME_ARGS="" + + # 模型目录 + if [ -n "$CUSTOM_MODELS" ]; then + MODEL_DIR="$CUSTOM_MODELS" + fi + + if [ -d "$MODEL_DIR" ]; then + VOLUME_ARGS="$VOLUME_ARGS -v ${MODEL_DIR}:${LOCAL_MODEL_DIR}" + print_info "模型目录：${MODEL_DIR} -> ${LOCAL_MODEL_DIR}" + else + print_warning "模型目录不存在：${MODEL_DIR}" + print_info "创建模型目录..." + mkdir -p "$MODEL_DIR" + VOLUME_ARGS="$VOLUME_ARGS -v ${MODEL_DIR}:${LOCAL_MODEL_DIR}" + fi + + # 配置文件 + if [ -n "$CUSTOM_CONFIG" ]; then + CONFIG_FILE="$CUSTOM_CONFIG" + fi + + if [ -f "$CONFIG_FILE" ]; then + VOLUME_ARGS="$VOLUME_ARGS -v ${CONFIG_FILE}:/etc/vllm/model_config.yaml" + print_info "配置文件：${CONFIG_FILE}" + else + print_warning "配置文件不存在：${CONFIG_FILE}" + print_info "将使用容器内默认配置" + fi + + # 端口设置 + PORT_ARG="-p 8000:8000" + if [ -n "$CUSTOM_PORT" ]; then + PORT_ARG="-p ${CUSTOM_PORT}:8000" + fi + + # 运行模式 + RUN_MODE="-d" + if [ "$DETACH" = false ]; then + RUN_MODE="-it" + fi + + # 构建运行命令 + print_info "启动容器..." + + docker run ${RUN_MODE} \ + --name "${CONTAINER_NAME}" \ + --device /dev/dri \ + --device /dev/kfd \ + --group-add video \ + --group-add render \ + --security-opt seccomp=unconfined \ + ${VOLUME_ARGS} \ + ${PORT_ARG} \ + -e LOCAL_MODEL_DIR="${LOCAL_MODEL_DIR}" \ + "${IMAGE_NAME}:${IMAGE_TAG}" + + if [ $? -eq 0 ]; then + print_success "容器启动成功" + + if [ "$DETACH" = true ]; then + echo "" + print_info "容器信息:" + echo " 容器名称：${CONTAINER_NAME}" + echo " 镜像：${IMAGE_NAME}:${IMAGE_TAG}" + echo " 状态：运行中" + echo " API 地址：http://localhost:8000" + echo "" + print_info "常用命令:" + echo " 查看日志：docker logs -f ${CONTAINER_NAME}" + echo " 停止容器：docker stop ${CONTAINER_NAME}" + echo " 启动容器：docker start ${CONTAINER_NAME}" + echo " 删除容器：docker rm -f ${CONTAINER_NAME}" + echo " 进入容器：docker exec -it ${CONTAINER_NAME} /bin/bash" + fi + else + print_error "容器启动失败" + exit 1 + fi +} + +# 主流程 +main() { + echo "==========================================" + echo "vLLM R9700 Container 构建和部署" + echo "==========================================" + echo "开始时间：$(date '+%Y-%m-%d %H:%M:%S')" + echo "" + + # 检查 Docker + check_docker + + # 停止旧容器 + stop_container + + # 构建镜像 + build_image + + # 如果仅构建，退出 + if [ "$BUILD_ONLY" = true ]; then + print_success "镜像构建完成" + print_info "运行以下命令启动容器:" + echo " $0" + exit 0 + fi + + # 运行容器 + run_container + + echo "" + echo "==========================================" + print_success "部署完成" + echo "==========================================" + echo "完成时间：$(date '+%Y-%m-%d %H:%M:%S')" +} + +# 执行主流程 +main diff --git a/model_config.yaml.example b/model_config.yaml.example new file mode 100644 index 0000000..2b2a37b --- /dev/null +++ b/model_config.yaml.example @@ -0,0 +1,110 @@ +# model_config.yaml - 多模型配置示例 +# 默认启动的模型 +default: "deepseek_r1_distill_qwen_32b_awq" + +# 模型配置 +models: + deepseek_r1_distill_qwen_14b: + path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-14B" + name: "DeepSeek-R1-Distill-Qwen-14B" + max_model_len: 8192 + gpu_memory_utilization: 0.9 + port: 2001 + dtype: "float16" + quantization: "awq" + tensor_parallel_size: 1 + enforce_eager: true + api_key: "sk-14b-20240101-abcdef123456" + + deepseek_r1_distill_qwen_32b_awq: + path: "/app/vllm/models/DeepSeek-R1-Distill-Qwen-32B-AWQ" + name: "DeepSeek-R1-Distill-Qwen-32B-AWQ" + + # 模型性能参数 + max_model_len: 32768 + gpu_memory_utilization: 0.95 + enforce_eager: true + max_num_seqs: 2 + max_num_batched_tokens: 1024 + block_size: 16 + tensor_parallel_size: 1 + swap_space: 0 + + # 新增：采样参数默认值 + sampling_defaults: + temperature: 0.6 + max_tokens: 4096 + top_p: 0.9 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: + - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + + # 其他配置 + dtype: "auto" + quantization: "awq" + port: 2001 + api_key: "sk-32b-20240101-ghijk789012" + + glm_4_7_flash_awq: + path: "/app/vllm/models/GLM-4.7-Flash-AWQ" + name: "GLM-4.7-Flash-AWQ" + + # 模型性能参数 + max_model_len: 32768 + gpu_memory_utilization: 0.9 + enforce_eager: false + max_num_seqs: 2 + max_num_batched_tokens: 1024 + block_size: 16 + tensor_parallel_size: 1 + swap_space: 0 + + # 新增：采样参数默认值 + sampling_defaults: + temperature: 0.6 + max_tokens: 4096 + top_p: 0.9 + frequency_penalty: 0.0 + presence_penalty: 0.0 + stop: + - "用户:" + - "助手:" + - "###" + - "问题:" + - "回答:" + + # 其他配置 + dtype: "auto" + quantization: "awq" + port: 2001 + api_key: "sk-32b-20240101-ghijk789012" + + qwen3_vl_32b_instruct_awq: + path: "/app/vllm/models/Qwen3-VL-32B-Instruct-AWQ" + name: "Qwen3-VL-32B-Instruct-AWQ" + max_model_len: 32768 + gpu_memory_utilization: 0.7 + port: 2001 + dtype: "auto" + quantization: "awq" + tensor_parallel_size: 1 + enforce_eager: true + api_key: "sk-glm-20240101-lmnop345678" + +# 服务器通用设置 +server: + host: "0.0.0.0" + log_level: "info" + # 全局管理员密钥（拥有所有模型的访问权限） + admin_key: "sk-admin-20240101-xyz789" + # 允许的请求头名称（支持多个，按顺序检查） + api_key_headers: ["Authorization", "X-API-Key", "api-key"] + # 是否允许通过查询参数传递密钥 + allow_query_param: true + # 查询参数名称 + api_key_param: "api_key" diff --git a/params_config.yaml b/params_config.yaml new file mode 100644 index 0000000..fd3da40 --- /dev/null +++ b/params_config.yaml @@ -0,0 +1,269 @@ +# vLLM 参数配置文件 +# 用于定义和识别 vLLM 服务器的各类参数 + +# 模型参数配置 +model_params: + # 必需参数 + required: + - path # 模型路径 + + # 可选参数 + optional: + - name # 模型名称 + - dtype # 数据类型 (auto, float16, float32, bfloat16) + - quantization # 量化方式 (awq, gptq, squeezellm) + - trust_remote # 是否信任远程代码 + +# 性能参数配置 +performance_params: + # GPU 相关 + gpu: + - tensor_parallel_size # 张量并行度 + - pipeline_parallel_size # 流水线并行度 + - gpu_memory_utilization # GPU 内存利用率 (0.0-1.0) + - swap_space # CPU 交换空间大小 (GB) + - max_num_batched_tokens # 最大批量 tokens 数 + - max_num_seqs # 最大并发请求数 + - num_scheduler_steps # 调度器步数 + + # 内存管理 + memory: + - block_size # 块大小 (8, 16, 32) + - max_model_len # 最大模型上下文长度 + - max_logprobs # 最大 logprobs 数 + - disable_sliding_window # 禁用滑动窗口 + +# 服务参数配置 +server_params: + # 网络配置 + network: + - host # 服务器主机地址 + - port # 服务器端口 + - ssl_keyfile # SSL 密钥文件 + - ssl_certfile # SSL 证书文件 + - ssl_ca_certs # SSL CA 证书 + - ssl_keyfile_password # SSL 密钥密码 + + # API 配置 + api: + - api_key # API 密钥 + - allowed_origins # 允许的源 + - timeout_keep_alive # 保持连接超时时间 + +# 采样参数配置 +sampling_params: + # 温度控制 + - temperature # 温度 (0.0-2.0) + - top_p # 核采样参数 (0.0-1.0) + - top_k # Top-K 采样 + - min_p # 最小概率 + + # 惩罚参数 + - frequency_penalty # 频率惩罚 (-2.0 到 2.0) + - presence_penalty # 存在惩罚 (-2.0 到 2.0) + - repetition_penalty # 重复惩罚 + + # 生成长度 + - max_tokens # 最大生成 tokens 数 + - min_tokens # 最小生成 tokens 数 + - stop # 停止词列表 + - stop_token_ids # 停止 token IDs + + # 其他采样选项 + - seed # 随机种子 + - use_beam_search # 使用束搜索 + - best_of # 束搜索的最佳候选数 + - length_penalty # 长度惩罚 + - early_stopping # 早期停止 + - ignore_eos # 忽略 EOS token + - skip_special_tokens # 跳过特殊 tokens + - spaces_between_special_tokens # 特殊 tokens 之间的空格 + +# 日志参数配置 +logging_params: + - log_level # 日志级别 (debug, info, warning, error) + - log_requests # 是否记录请求 + - log_responses # 是否记录响应 + +# 高级参数配置 +advanced_params: + # 执行模式 + - enforce_eager # 强制使用 eager 模式 + - cuda_graphs # CUDA 图 + - use_v2_block_manager # 使用 V2 块管理器 + + # 注意力后端 + - use_rocm_attn # 使用 ROCm 注意力后端 + - attention_backend # 注意力后端类型 + + # 分布式 + - distributed_executor_backend # 分布式执行器后端 + - ray_workers_use_nsight # Ray workers 使用 nsight + + # 其他 + - load_format # 加载格式 + - download_dir # 下载目录 + - revision # 模型版本 + - code_revision # 代码版本 + - tokenizer_revision # tokenizer 版本 + +# 参数类型映射 +param_types: + # 整数类型 + integer: + - tensor_parallel_size + - pipeline_parallel_size + - max_num_batched_tokens + - max_num_seqs + - block_size + - max_model_len + - max_tokens + - min_tokens + - top_k + - best_of + - seed + - num_scheduler_steps + - max_logprobs + - swap_space + - timeout_keep_alive + + # 浮点数类型 + float: + - gpu_memory_utilization + - temperature + - top_p + - min_p + - frequency_penalty + - presence_penalty + - repetition_penalty + - length_penalty + + # 布尔类型 + boolean: + - trust_remote + - enforce_eager + - use_rocm_attn + - use_beam_search + - early_stopping + - ignore_eos + - skip_special_tokens + - spaces_between_special_tokens + - log_requests + - log_responses + - disable_sliding_window + - use_v2_block_manager + + # 字符串类型 + string: + - path + - name + - dtype + - quantization + - host + - port + - api_key + - ssl_keyfile + - ssl_certfile + - ssl_ca_certs + - ssl_keyfile_password + - log_level + - attention_backend + - load_format + - download_dir + - revision + - code_revision + - tokenizer_revision + + # 列表类型 + list: + - stop + - stop_token_ids + - allowed_origins + +# 参数默认值 +param_defaults: + host: "0.0.0.0" + port: 8000 + dtype: "auto" + gpu_memory_utilization: 0.9 + max_model_len: 8192 + tensor_parallel_size: 1 + max_num_seqs: 256 + block_size: 16 + temperature: 0.7 + top_p: 0.9 + max_tokens: 256 + enforce_eager: false + trust_remote: false + log_level: "info" + +# 参数验证规则 +param_validation: + gpu_memory_utilization: + min: 0.0 + max: 1.0 + + temperature: + min: 0.0 + max: 2.0 + + top_p: + min: 0.0 + max: 1.0 + + min_p: + min: 0.0 + max: 1.0 + + frequency_penalty: + min: -2.0 + max: 2.0 + + presence_penalty: + min: -2.0 + max: 2.0 + + tensor_parallel_size: + min: 1 + max: 8 + + block_size: + allowed_values: [8, 16, 32] + +# 参数分组（用于配置文件组织） +param_groups: + basic: + name: "基础配置" + params: + - path + - name + - port + - api_key + + performance: + name: "性能配置" + params: + - tensor_parallel_size + - gpu_memory_utilization + - max_model_len + - max_num_seqs + - block_size + + sampling: + name: "采样配置" + params: + - temperature + - top_p + - max_tokens + - frequency_penalty + - presence_penalty + - stop + + advanced: + name: "高级配置" + params: + - dtype + - quantization + - enforce_eager + - trust_remote + - swap_space diff --git a/scripts/01-rocm-envs.sh b/scripts/01-rocm-envs.sh new file mode 100644 index 0000000..eada83f --- /dev/null +++ b/scripts/01-rocm-envs.sh @@ -0,0 +1,4 @@ +export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 +export FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE" +export VLLM_TARGET_DEVICE=rocm +export VLLM_USE_TRITON_AWQ=1 \ No newline at end of file diff --git a/scripts/99-toolbox-banner.sh b/scripts/99-toolbox-banner.sh new file mode 100644 index 0000000..67085ea --- /dev/null +++ b/scripts/99-toolbox-banner.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Lightweight banner with machine/GPU and ROCm version (vLLM edition) +# No Triton env sourcing, same info/format as the image/video banner. + +# Only show for interactive shells +case $- in *i*) ;; *) return 0 ;; esac + +oem_info() { + local v="" m="" d lv lm + for d in /sys/class/dmi/id /sys/devices/virtual/dmi/id; do + [[ -r "$d/sys_vendor" ]] && v=$(<"$d/sys_vendor") + [[ -r "$d/product_name" ]] && m=$(<"$d/product_name") + [[ -n "$v" || -n "$m" ]] && break + done + # ARM/SBC fallback + if [[ -z "$v" && -z "$m" && -r /proc/device-tree/model ]]; then + tr -d '\0' /dev/null 2>&1; then + name=$(rocm-smi --showproductname --csv 2>/dev/null | tail -n1 | cut -d, -f2) + [[ -z "$name" ]] && name=$(rocm-smi --showproductname 2>/dev/null | grep -m1 -E 'Product Name|Card series' | sed 's/.*: //') + fi + if [[ -z "$name" ]] && command -v rocminfo >/dev/null 2>&1; then + name=$(rocminfo 2>/dev/null | awk -F': ' '/^[[:space:]]*Name:/{print $2; exit}') + fi + if [[ -z "$name" ]] && command -v lspci >/dev/null 2>&1; then + name=$(lspci -nn 2>/dev/null | grep -Ei 'vga|display|gpu' | grep -i amd | head -n1 | cut -d: -f3-) + fi + # trim + name=$(printf '%s' "$name" | sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+$//' -e 's/[[:space:]]\{2,\}/ /g') + printf '%s\n' "${name:-Unknown AMD GPU}" +} + +rocm_version() { + # Prefer the PyTorch HIP version from the venv, fallback to rocm pkg metadata + local PY="/torch-therock/.venv/bin/python" + [[ -x "$PY" ]] || PY="python" + "$PY" - <<'PY' 2>/dev/null || true +try: + import torch + v = getattr(getattr(torch, "version", None), "hip", "") or "" + if v: + print(v) + else: + raise Exception("no torch.version.hip") +except Exception: + try: + import importlib.metadata as im + try: + print(im.version("_rocm_sdk_core")) + except Exception: + print(im.version("rocm")) + except Exception: + print("") +PY +} + +MACHINE="$(oem_info)" +GPU="$(gpu_name)" +ROCM_VER="$(rocm_version)" + +echo +cat <<'ASCII' + _____ _____ ______ ____ _ _ ___ ______ ___ ___ +| __ \ /\ | __ \| ____/ __ \| \ | | / _ \____ / _ \ / _ \ +| |__) | / \ | | | | |__ | | | | \| | | (_) | / / | | | | | | +| _ / / /\ \ | | | | __|| | | | . ` | \__, | / /| | | | | | | +| | \ \ / ____ \| |__| | |___| |__| | |\ | / / / / | |_| | |_| | +|_| \_\/_/ \_\_____/|______\____/|_| \_| /_/ /_/ \___/ \___/ + + _____ _____ _____ ____ + /\ |_ _| | __ \| __ \ / __ \ + / \ | | | |__) | |__) | | | | + / /\ \ | | | ___/| _ /| | | | + / ____ \ _| |_ | | | | \ \| |__| | + /_/ \_\_____| |_| |_| \_\\____/ + + v L L M +ASCII +echo +printf 'AMD R9700 — vLLM Toolbox (gfx1201, ROCm via TheRock)\n' +[[ -n "$ROCM_VER" ]] && printf 'ROCm nightly: %s\n' "$ROCM_VER" +echo +printf 'Machine: %s\n' "$MACHINE" +printf 'GPU : %s\n\n' "$GPU" +printf 'Repo : https://github.com/kyuz0/amd-r9700-vllm-toolboxes\n' +printf 'Image : docker.io/kyuz0/vllm-therock-gfx1201:latest\n\n' +printf 'Included:\n' +printf ' - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling" +printf ' - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct" +printf ' - %-16s → %s\n' "API test" "curl localhost:8000/v1/chat/completions" +echo +printf 'SSH tip: ssh -L 8000:localhost:8000 user@host\n\n' + +unset PROMPT_COMMAND +PS1='\u@\h:\w\$ ' diff --git a/scripts/start_vllm.py b/scripts/start_vllm.py new file mode 100644 index 0000000..a5d8e81 --- /dev/null +++ b/scripts/start_vllm.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import yaml +import shutil +import subprocess +from pathlib import Path + +# Add benchmarks dir to path to import config +SCRIPT_DIR = Path(__file__).parent.resolve() +OPT_DIR = Path("/opt") + +# Required environment variable pointing to a local models directory +LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR") +if not LOCAL_MODEL_DIR: + print("Error: LOCAL_MODEL_DIR environment variable is required.") + sys.exit(1) + +# Configuration file path +CONFIG_FILE = os.getenv("VLLM_CONFIG_FILE", "/etc/vllm/model_config.yaml") + +# Default configuration +DEFAULT_CONFIG = { + "default": "", + "models": {}, + "server": { + "host": "0.0.0.0", + "log_level": "info" + } +} + +def detect_gpus(): + """Detects AMD GPUs via rocm-smi or /dev/dri.""" + try: + # Try rocm-smi first + res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode == 0: + count = res.stdout.count("GPU") + if count > 0: return count + except: pass + + # Fallback to /dev/dri/render* + try: + return len(list(Path("/dev/dri").glob("renderD*"))) + except: + return 1 + +def load_config(): + """Load configuration from YAML file.""" + config = DEFAULT_CONFIG.copy() + + if Path(CONFIG_FILE).exists(): + try: + with open(CONFIG_FILE, "r", encoding="utf-8") as f: + user_config = yaml.safe_load(f) + if user_config: + config.update(user_config) + except Exception as e: + print(f"Warning: Failed to load config file: {e}") + print("Using default configuration.") + else: + print(f"Warning: Config file not found at {CONFIG_FILE}") + print("Using default configuration.") + + return config + +def nuke_vllm_cache(): + """Removes vLLM cache directory to fix potential graph/incompatibility issues.""" + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + print(f"Clearing vLLM cache at {cache}...", end="", flush=True) + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + print(" Done.") + except Exception as e: + print(f" Failed: {e}") + +def get_local_model_path(model_name): + """Get local model path from LOCAL_MODEL_DIR.""" + # Try multiple paths: + # 1) LOCAL_MODEL_DIR/model_name + # 2) case-insensitive match in LOCAL_MODEL_DIR + + # Exact match + candidate_exact = os.path.join(LOCAL_MODEL_DIR, model_name) + if os.path.isdir(candidate_exact): + return candidate_exact + + # Case-insensitive match + try: + for entry in os.listdir(LOCAL_MODEL_DIR): + if entry.lower() == model_name.lower(): + entry_path = os.path.join(LOCAL_MODEL_DIR, entry) + if os.path.isdir(entry_path): + return entry_path + except Exception as e: + print(f"Error searching for model: {e}") + + return None + +def verify_model_path(model_path): + """Verify that the model path contains required files.""" + required = ["config.json", "pytorch_model.bin", "model.safetensors"] + found = any(os.path.isfile(os.path.join(model_path, f)) for f in required) + if not found: + print(f"Error: local model dir {model_path} missing expected files {required}") + return False + return True + +def main(): + # Load configuration + config = load_config() + + # Get model name from command line or use default + model_name = None + if len(sys.argv) > 1: + model_name = sys.argv[1] + + if not model_name: + model_name = config.get("default", "") + if not model_name: + print("Error: Either specify a model name as argument or set 'default' in config.") + sys.exit(1) + + # Get model configuration + model_config = config.get("models", {}).get(model_name) + if not model_config: + print(f"Error: Model '{model_name}' not found in configuration.") + sys.exit(1) + + # Get model path + model_path = model_config.get("path") + if not model_path: + # Fallback to LOCAL_MODEL_DIR if path not specified + model_path = get_local_model_path(model_name) + if not model_path: + print(f"Error: model '{model_name}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}") + sys.exit(1) + + # Verify model path + if not verify_model_path(model_path): + sys.exit(1) + + # Detect GPU count + gpu_count = detect_gpus() + + # Get server configuration + server_config = config.get("server", {}) + host = server_config.get("host", "0.0.0.0") + + # Build command + cmd = ["vllm", "serve", model_path] + + # Add server parameters + cmd.extend(["--host", host]) + if "port" in model_config: + cmd.extend(["--port", str(model_config["port"])]) + + # Add API key for OpenAI compatibility + api_key = model_config.get("api_key") + if api_key: + cmd.extend(["--api-key", api_key]) + + # Add model parameters + if "tensor_parallel_size" in model_config: + tp_size = min(model_config["tensor_parallel_size"], gpu_count) + cmd.extend(["--tensor-parallel-size", str(tp_size)]) + + if "max_num_seqs" in model_config: + cmd.extend(["--max-num-seqs", str(model_config["max_num_seqs"])]) + + if "max_model_len" in model_config: + cmd.extend(["--max-model-len", str(model_config["max_model_len"])]) + + if "gpu_memory_utilization" in model_config: + cmd.extend(["--gpu-memory-utilization", str(model_config["gpu_memory_utilization"])]) + + if "dtype" in model_config: + cmd.extend(["--dtype", model_config["dtype"]]) + else: + cmd.extend(["--dtype", "auto"]) + + if "max_num_batched_tokens" in model_config: + cmd.extend(["--max-num-batched-tokens", str(model_config["max_num_batched_tokens"])]) + + if "block_size" in model_config: + cmd.extend(["--block-size", str(model_config["block_size"])]) + + if "swap_space" in model_config: + cmd.extend(["--swap-space", str(model_config["swap_space"])]) + + if model_config.get("enforce_eager", False): + cmd.append("--enforce-eager") + + if model_config.get("trust_remote", False): + cmd.append("--trust-remote-code") + + # Set environment variables + env = os.environ.copy() + + if model_config.get("use_rocm_attn", False): + env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1" + env["VLLM_USE_TRITON_FLASH_ATTN"] = "0" + + # Clear cache if requested + if model_config.get("clear_cache", False): + nuke_vllm_cache() + + # Print configuration + print("\n" + "="*60) + print(f" Launching: {model_name}") + print(f" Model Path: {model_path}") + print(f" Host: {host}") + if "port" in model_config: + print(f" Port: {model_config['port']}") + if api_key: + print(f" API Key: {api_key[:8]}...{api_key[-4:]}") + if "tensor_parallel_size" in model_config: + print(f" TP Size: {tp_size}") + if "max_num_seqs" in model_config: + print(f" Max Seqs: {model_config['max_num_seqs']}") + if "max_model_len" in model_config: + print(f" Max Ctx: {model_config['max_model_len']}") + if "gpu_memory_utilization" in model_config: + print(f" GPU Util: {model_config['gpu_memory_utilization']}") + print(f" Command: {' '.join(cmd)}") + print("="*60 + "\n") + + # Launch vLLM server + os.execvpe("vllm", cmd, env) + +if __name__ == "__main__": + main() diff --git a/scripts/zz-venv-last.sh b/scripts/zz-venv-last.sh new file mode 100644 index 0000000..eee3228 --- /dev/null +++ b/scripts/zz-venv-last.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Ensure /opt/venv/bin is first even if ~/.local/bin or ~/.cargo/bin prepend themselves via user dotfiles. + +_venv_path_fix() { + # remove any existing /opt/venv/bin entries, then prepend one + local newpath + newpath="$(printf '%s' "$PATH" | awk -v RS=: -v ORS=: '$0!="/opt/venv/bin"{print}')" + PATH="/opt/venv/bin:${newpath%:}" +} + +# run once after shell init; don't duplicate +case "$PROMPT_COMMAND" in + *_venv_path_fix*) : ;; + *) PROMPT_COMMAND="_venv_path_fix${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;; +esac +