diff --git a/docker/Dockerfile b/docker/Dockerfile index c0a91ac..a4d5f22 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,11 @@ # ============================================================================= -# MinerU on ROCm 7.2.1 Docker Image +# MinerU on ROCm 7.2.1 Docker Image（HK 云服务器 / 海外版） # 原生 Linux + Ubuntu 24.04 + ROCm 7.2.1 + PyTorch 2.11.0 + vllm + MinerU 3.2.0 -# -# 国内网络优化版：Ubuntu/PyPI/GitHub 全部使用国内镜像 -# 构建前请根据你的 GPU 修改 ARCH 参数（默认 gfx1201 = RX 9070） # ============================================================================= FROM ubuntu:24.04 # -- 构建参数 --------------------------------------------------------------- -# GPU 架构：gfx1201(RX 9070) gfx1200(RX 9060) gfx1100(RX 7900) gfx1101(RX 7800/7700) gfx1030(RX 6900/6800) ARG ARCH=gfx1201 ARG PYTHON_VER=3.12 ARG VENV=/opt/mineru_venv @@ -18,8 +14,7 @@ ARG TORCH_INDEX=https://download.pytorch.org/whl/rocm7.2 # -- GitHub 访问（国内无镜像，需代理）---------------------------------------- ARG GIT_PROXY=http://127.0.0.1:8118 -# -- 国内镜像配置 ----------------------------------------------------------- -# PyPI 镜像 +# -- 国内镜像 ---------------------------------------------------------------- ARG PIP_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple # -- 环境变量 --------------------------------------------------------------- @@ -37,45 +32,37 @@ WORKDIR /opt # =========================================================================== # 阶段 1：换国内源 + 安装 ROCm 7.2.1 # =========================================================================== -# Ubuntu 24.04 使用 deb822 格式，默认源文件是 /etc/apt/sources.list.d/ubuntu.sources RUN sed -i 's|http://.*archive.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources && \ sed -i 's|http://.*security.ubuntu.com|http://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && apt-get install -y --no-install-recommends \ wget curl ca-certificates gnupg software-properties-common && \ - # 添加 AMD ROCm 仓库（repo.radeon.com 通常国内可直连） wget -q https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ gpg --dearmor | tee /etc/apt/trusted.gpg.d/rocm.gpg > /dev/null && \ echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/7.2.1 noble main' \ > /etc/apt/sources.list.d/rocm.list && \ - # apt pinning：AMD 仓库优先级高于 Ubuntu 自带（避免拿到旧版 rocminfo） printf 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600\n' \ > /etc/apt/preferences.d/rocm-pin-600 && \ apt-get update && \ - # 安装 ROCm 基础组件（apt pinning 确保从 AMD 仓库拉） apt-get install -y --no-install-recommends \ rocminfo rocm-device-libs hip-dev miopen-hip && \ - # 清理 apt-get clean && rm -rf /var/lib/apt/lists/* # =========================================================================== -# 阶段 2：ROCm 头文件补丁（LLVM 22 兼容性修复） -# 这些是 ROCm 7.2.1 在 24.04 上的已知问题，每次 apt 升级 ROCm 后需重新应用 +# 阶段 2：ROCm 头文件补丁 # =========================================================================== RUN set -ex && \ - # 补丁 1: hipcc/clang 符号链接（hipcc.pl 硬编码 clang-17，实际是 clang-22） + # 补丁 1: hipcc/clang 符号链接 ln -sf /usr/bin/hipvars.pm /usr/share/perl5/hipvars.pm && \ - # hipcc.pl 在 WSL2 存在，原生 Linux 不存在；不存在时保留 apt 安装的版本 ([ -f /usr/bin/hipcc.pl ] && ln -sf /usr/bin/hipcc.pl /opt/rocm/bin/hipcc) || echo "hipcc.pl not found, keeping apt-installed hipcc" && \ ln -sf /opt/rocm/llvm/bin/clang-22 /opt/rocm/llvm/bin/clang-17 && \ ln -sf /opt/rocm/llvm/bin/clang++ /opt/rocm/llvm/bin/clang++-17 && \ # 补丁 2: __hip_internal::conditional → std::conditional find /opt/rocm/include/hip -name "*.h" \ -exec sed -i 's/__hip_internal::conditional/std::conditional/g' {} + && \ - # 补丁 3: warpSize 常量（__AMDGCN_WAVEFRONT_SIZE 在 LLVM 22 未定义） + # 补丁 3: warpSize 常量 find /opt/rocm/include/hip -name "amd_warp_functions.h" \ -exec sed -i 's/static constexpr int warpSize = __AMDGCN_WAVEFRONT_SIZE;/constexpr int warpSize = 32;/g' {} + && \ # 补丁 4: __activemask() → __builtin_amdgcn_read_exec() - # 注意：只改 amd_warp_sync_functions.h，不要动 amd_warp_functions.h（那是定义本身） sed -i 's/__activemask()/__builtin_amdgcn_read_exec()/g' \ /opt/rocm/include/hip/amd_detail/amd_warp_sync_functions.h && \ echo "ROCm 7.2.1 header patches applied." @@ -87,16 +74,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential git ninja-build pkg-config \ python${PYTHON_VER} python${PYTHON_VER}-venv python${PYTHON_VER}-dev \ libnuma-dev libdrm2 libhwloc-dev libgl1 \ - # vllm 运行时依赖 libgomp1 libopenblas0 && \ apt-get clean && rm -rf /var/lib/apt/lists/* # =========================================================================== -# 阶段 4：CMake 4.0（vllm 要求 ≥ 4.0，Ubuntu 24.04 自带 3.28 不够） +# 阶段 4：CMake 4.0 # =========================================================================== RUN cd /tmp && \ - # wget 通过代理下载 CMake（国内 GitHub 不通） - export http_proxy=${GIT_PROXY} https_proxy=${GIT_PROXY} && \ wget -q https://github.com/Kitware/CMake/releases/download/v4.0.0/cmake-4.0.0-linux-x86_64.tar.gz && \ tar -xzf cmake-4.0.0-linux-x86_64.tar.gz && \ cp -r cmake-4.0.0-linux-x86_64/bin/* /usr/local/bin/ && \ @@ -108,22 +92,19 @@ RUN cd /tmp && \ # 阶段 5：Python 虚拟环境 + PyTorch ROCm # =========================================================================== RUN python${PYTHON_VER} -m venv ${VENV} && \ - # 配置 pip 国内镜像 mkdir -p /root/.pip && \ echo "[global]" > /root/.pip/pip.conf && \ echo "index-url = ${PIP_INDEX}" >> /root/.pip/pip.conf && \ ${VENV}/bin/pip install --no-cache-dir -U pip setuptools wheel && \ - # 安装 PyTorch ROCm 版（指定 index-url 覆盖全局镜像） ${VENV}/bin/pip install --no-cache-dir --pre \ torch==2.11.0+rocm7.2 \ torchvision \ pytorch-triton-rocm \ --index-url ${TORCH_INDEX} && \ - # 验证 - ${VENV}/bin/python -c "import torch; print('PyTorch:', torch.__version__); print('ROCm:', torch.version.hip); assert torch.version.hip is not None" + ${VENV}/bin/python -c "import torch; print('PyTorch:', torch.__version__); assert torch.version.hip is not None" # =========================================================================== -# 阶段 6：ROCm 开发包（vllm 编译必需） +# 阶段 6：ROCm 开发包 # =========================================================================== RUN apt-get update && apt-get install -y --no-install-recommends \ hipblas-dev hiprand-dev hipsparse-dev hipsparselt-dev \ @@ -133,7 +114,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ apt-get clean && rm -rf /var/lib/apt/lists/* # =========================================================================== -# 阶段 6.5：安装 amdsmi（原生 Linux 必需，vllm 平台检测依赖） +# 阶段 6.5：安装 amdsmi # =========================================================================== RUN if [ -d /opt/rocm/share/amd_smi ]; then \ cp -r /opt/rocm/share/amd_smi /opt/amd_smi && \ @@ -144,35 +125,27 @@ RUN if [ -d /opt/rocm/share/amd_smi ]; then \ fi # =========================================================================== -# 阶段 7：amd-aiter + flash_attn（宿主机预克隆，COPY 进镜像） +# 阶段 7：amd-aiter + flash_attn # =========================================================================== -# 构建前请先在宿主机 docker/ 目录下克隆： -# git clone --depth 1 --recursive https://github.com/ROCm/aiter.git -# git clone --depth 1 https://github.com/Dao-AILab/flash-attention.git -# cd flash-attention && git checkout bba578d43974c1d3ba157ab597124dd0fe2ccdb4 -# git submodule update --init --depth 1 && cd .. -# =========================================================================== -COPY aiter/ /opt/aiter/ -COPY flash-attention/ /opt/flash-attention/ RUN set -ex && \ - # aiter（AMD 优化的 attention 算子） + git config --global http.proxy ${GIT_PROXY} && \ + git config --global https.proxy ${GIT_PROXY} && \ + cd /opt && git clone --recursive --depth 1 https://github.com/ROCm/aiter.git && \ ${VENV}/bin/pip install --no-cache-dir -e /opt/aiter && \ - # flash_attn（Triton AMD 后端，锁定已验证的 commit） + cd /opt && git clone --recursive https://github.com/Dao-AILab/flash-attention.git && \ + cd flash-attention && git checkout bba578d43974c1d3ba157ab597124dd0fe2ccdb4 && \ ${VENV}/bin/pip install --no-cache-dir --no-build-isolation -e /opt/flash-attention && \ - # 验证 PyTorch 没被覆盖 ${VENV}/bin/python -c "import torch; v=torch.__version__; assert 'rocm' in v, f'PyTorch overwritten: {v}'; print('PyTorch OK:', v)" # =========================================================================== -# 阶段 8a：编译 vllm（宿主机预克隆 + COPY，cmake + ninja，最耗时，单独一层缓存） -# 构建前请在宿主机 docker/ 目录下克隆： -# git clone --depth 1 https://github.com/vllm-project/vllm.git +# 阶段 8a：编译 vllm（cmake + ninja，最耗时，单独缓存） # =========================================================================== -COPY vllm/ /opt/vllm/ RUN set -ex && \ - # setuptools 升级（PEP 639 兼容） + git config --global http.proxy ${GIT_PROXY} && \ + git config --global https.proxy ${GIT_PROXY} && \ ${VENV}/bin/pip install --no-cache-dir -U \ "setuptools>=77.0.3" setuptools_scm setuptools_rust wheel && \ - # 补丁 5：注释掉 vllm mamba 模块的 operator+ 定义（可能已被 vllm 移除） + cd /opt && git clone --depth 1 https://github.com/vllm-project/vllm.git && \ cd /opt/vllm && \ if [ -f csrc/mamba/mamba_ssm/selective_scan.h ]; then \ sed -i '109,121s/^/\/\/ /' csrc/mamba/mamba_ssm/selective_scan.h && \ @@ -180,14 +153,12 @@ RUN set -ex && \ else \ echo "vllm mamba selective_scan.h not found (upstream removed), skipping patch."; \ fi && \ - # cmake 别名兜底 mkdir -p /opt/rocm/lib/cmake/hiprand && \ printf 'include(/opt/rocm/lib/cmake/rocrand/rocrand-config.cmake)\nif(TARGET roc::rocrand AND NOT TARGET hip::hiprand)\n add_library(hip::hiprand ALIAS roc::rocrand)\nendif()\n' \ > /opt/rocm/lib/cmake/hiprand/hiprand-config.cmake && \ mkdir -p /opt/rocm/lib/cmake/hipblas && \ printf 'include(/opt/rocm/lib/cmake/rocblas/rocblas-config.cmake)\nif(TARGET roc::rocblas AND NOT TARGET hip::hipblas)\n add_library(hip::hipblas ALIAS roc::rocblas)\nendif()\n' \ > /opt/rocm/lib/cmake/hipblas/hipblas-config.cmake && \ - # cmake 配置 mkdir -p /opt/vllm_build && \ cmake -S /opt/vllm -B /opt/vllm_build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ @@ -200,36 +171,27 @@ RUN set -ex && \ -DHIP_COMPILER=/opt/rocm/llvm/bin/clang++ \ -DHIP_PATH=/opt/rocm \ -DCMAKE_PREFIX_PATH="/opt/rocm;${VENV}/lib/python${PYTHON_VER}/site-packages/torch/share/cmake" && \ - # ninja 编译（-j8，32GB 内存；若 < 16GB 请改为 -j2） cd /opt/vllm_build && ninja -j8 && \ - # 安装 .so 到 vllm 源码目录 cp /opt/vllm_build/*.abi3.so /opt/vllm/vllm/ && \ echo "vllm C++ build complete (layer cached)." # =========================================================================== -# 阶段 8b：安装 vllm + 验证 PyTorch（独立层，失败不影响 8a 缓存） +# 阶段 8b：安装 vllm + 平台补丁 + 验证 # =========================================================================== RUN set -ex && \ - # 安装 vllm（让 pip 解析运行时依赖：xgrammar, regex, compressed_tensors 等） cd /opt/vllm && ${VENV}/bin/pip install --no-cache-dir -e . --no-build-isolation && \ - # --no-build-isolation 可能漏装部分运行时依赖，显式补装 ${VENV}/bin/pip install --no-cache-dir regex && \ - # 应用 vllm 平台检测补丁（独立脚本，避免 Dockerfile 解析问题） ${VENV}/bin/python /opt/patch_vllm_platform.py && \ - # 验证 PyTorch 没被 vllm 依赖覆盖 - ${VENV}/bin/python -c "import torch; v=torch.__version__; assert 'rocm' in v, f'PyTorch overwritten by vllm deps: {v}'; print('PyTorch OK:', v)" && \ - # 先重装 ROCm PyTorch 覆盖可能的 CUDA 版，再清理 CUDA triton 元数据 + ${VENV}/bin/python -c "import torch; v=torch.__version__; assert 'rocm' in v, f'PyTorch overwritten: {v}'; print('PyTorch OK:', v)" && \ ${VENV}/bin/pip install --no-cache-dir --force-reinstall \ torch==2.11.0+rocm7.2 torchvision pytorch-triton-rocm \ --index-url ${TORCH_INDEX} && \ ${VENV}/bin/pip uninstall -y triton triton-rocm 2>/dev/null; \ - # 确保 vllm 基本可导入（构建时无 GPU，仅验证 import） ${VENV}/bin/python -c "import vllm; print('vllm import OK:', vllm.__version__)" && \ - # 清理构建目录（减小镜像体积，约 3-5GB） rm -rf /opt/vllm_build # =========================================================================== -# 阶段 8.5：复制辅助脚本（必须在 MinerU 安装前就位） +# 阶段 8.5：复制辅助脚本 # =========================================================================== COPY scripts/apply_mineru_patches.py /opt/apply_mineru_patches.py COPY scripts/cache_warmer.py /opt/cache_warmer.py @@ -239,20 +201,15 @@ COPY scripts/patch_vllm_platform.py /opt/patch_vllm_platform.py # 阶段 9：安装 MinerU + RDNA 适配补丁 # =========================================================================== RUN set -ex && \ - # pip 国内镜像已在阶段 5 全局配置 ${VENV}/bin/pip install --no-cache-dir 'mineru[core]' && \ - # 验证 PyTorch 没被覆盖 ${VENV}/bin/python -c "import torch; v=torch.__version__; assert 'rocm' in v, f'PyTorch overwritten: {v}'; print('PyTorch OK:', v)" && \ - # 应用 MinerU RDNA 适配补丁 ${VENV}/bin/python /opt/apply_mineru_patches.py # =========================================================================== # 阶段 10：入口与最终验证 # =========================================================================== RUN echo 'source /opt/mineru_venv/bin/activate' >> /etc/bash.bashrc && \ - echo "MinerU Docker image built successfully." && \ - ${VENV}/bin/python -c "import torch, vllm, mineru; print('='*50); print('MinerU ROCm Docker Image Ready'); print(f' PyTorch : {torch.__version__}'); print(f' ROCm : {torch.version.hip}'); print(f' vllm : {vllm.__version__}'); print(f' MinerU : {mineru.__version__}'); print(f' Arch : ${ARCH}'); print('='*50)" + ${VENV}/bin/python -c "import torch, vllm, mineru; print('='*50); print('MinerU ROCm Docker Image Ready'); print(f' PyTorch: {torch.__version__}'); print(f' vllm: {vllm.__version__}'); print(f' MinerU: {mineru.__version__}'); print('='*50)" -# 容器入口：默认 bash，用户可 override ENTRYPOINT ["/bin/bash", "-c"] CMD ["bash"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 95e84ab..30a27bf 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,56 +9,30 @@ services: build: context: . dockerfile: Dockerfile - network: host # 容器共享宿主机网络（构建时需要） + network: host args: - # ---- 按你的 GPU 修改 ---- - # gfx1201 = RX 9070 / 9070 XT / 9070 GRE - # gfx1200 = RX 9060 XT / 9060 XT LP - # gfx1100 = RX 7900 XTX / XT / GRE - # gfx1101 = RX 7800 XT / 7700 XT - # gfx1030 = RX 6950 / 6900 / 6800 系列 ARCH: gfx1201 + GIT_PROXY: http://127.0.0.1:8118 container_name: mineru-rocm stdin_open: true tty: true - ipc: host # vllm 共享内存需要 - - # ---- GPU 设备透传（必需）---- + ipc: host devices: - - /dev/kfd # ROCm KFD 内核驱动接口 - - /dev/dri # GPU 渲染节点 (renderD*) - - # ---- 安全配置（ROCm 需要）---- + - /dev/kfd + - /dev/dri security_opt: - - seccomp=unconfined # 允许 ROCm 系统调用 - - # ---- 用户权限（访问 GPU 设备）---- + - seccomp=unconfined group_add: - - video # /dev/dri/render* 权限 - - render # /dev/kfd 权限 - - # ---- 环境变量 ---- + - video + - render environment: - MINERU_MODEL_SOURCE=${MINERU_MODEL_SOURCE:-huggingface} - HF_HUB_CACHE=${HF_HUB_CACHE:-/opt/models/huggingface} - MODELSCOPE_CACHE=${MODELSCOPE_CACHE:-/opt/models/modelscope} - - # ---- 卷挂载 ---- volumes: - # 输入/输出目录（按需修改） - ${INPUT_DIR:-./data/input}:/data/input:ro - ${OUTPUT_DIR:-./data/output}:/data/output - # 模型缓存（持久化，避免每次下载） - ${MODEL_DIR:-./data/models}:/opt/models - # MIOpen kernel 缓存（持久化，避免每次预热） - ${MIOPEN_CACHE:-./data/miopen}:/root/.cache/miopen - - # ---- 启动命令（默认 bash，可改）---- command: bash - - # ---- 端口（WebUI / API 模式时启用）---- - # ports: - # - "7860:7860" # WebUI - # - "8000:8000" # API - restart: "no"