first commit
This commit is contained in:
Donato Capitella
@@ -0,0 +1,97 @@
|
||||
name: Build & Publish AMD R9700 Toolboxes
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
backends:
|
||||
description: >
|
||||
Comma-separated backends to build (e.g. "rocm-7beta,rocm-7rc").
|
||||
Use "all" to build everything.
|
||||
required: false
|
||||
default: all
|
||||
|
||||
env:
|
||||
DOCKERHUB_REPO: docker.io/kyuz0/amd-r9700-toolboxes
|
||||
LOCAL_PREFIX: llama
|
||||
|
||||
jobs:
|
||||
# 1) Prepare a clean JSON array for the matrix
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix_json: ${{ steps.mk.outputs.matrix_json }}
|
||||
steps:
|
||||
- id: mk
|
||||
shell: bash
|
||||
run: |
|
||||
# Input from the Run workflow form
|
||||
IN='${{ inputs.backends }}'
|
||||
|
||||
if [[ "$IN" == "all" || -z "$IN" ]]; then
|
||||
JSON='["rocm-6.4.4","rocm-6.4.4-rocwmma","rocm-7.1","rocm-7.1-rocwmma","rocm-7-nightly","rocm-7-nightly-rocwmma","rocm-7.9","rocm-7.9-rocwmma","vulkan-amdvlk","vulkan-radv"]'
|
||||
else
|
||||
# Remove spaces and build JSON array from comma list
|
||||
IN_CLEAN=$(echo "$IN" | tr -d '[:space:]')
|
||||
JSON='["'${IN_CLEAN//,/\",\"}'"]'
|
||||
fi
|
||||
|
||||
echo "matrix_json=${JSON}" >> "$GITHUB_OUTPUT"
|
||||
echo "Using matrix: ${JSON}"
|
||||
|
||||
# 2) Build each backend in parallel using the prepared matrix
|
||||
build-and-push:
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
backend: ${{ fromJson(needs.prepare.outputs.matrix_json) }}
|
||||
|
||||
steps:
|
||||
- name: Free up runner disk space
|
||||
run: |
|
||||
echo "Before cleanup:" && df -h /
|
||||
sudo rm -rf \
|
||||
/usr/share/dotnet \
|
||||
/usr/local/lib/android \
|
||||
/opt/ghc \
|
||||
/opt/hostedtoolcache/CodeQL
|
||||
docker system prune --all --force
|
||||
docker builder prune --all --force
|
||||
echo "After cleanup:" && df -h /
|
||||
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set build timestamp
|
||||
run: echo "BUILD_TS=$(date +%Y%m%dT%H%M%S)" >> $GITHUB_ENV
|
||||
|
||||
- name: Build & push ${{ matrix.backend }}
|
||||
working-directory: toolboxes
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
B="${{ matrix.backend }}"
|
||||
DF="Dockerfile.$B"
|
||||
NAME="${B}"
|
||||
LI="${LOCAL_PREFIX}-${NAME}"
|
||||
TAG="${NAME}_${BUILD_TS}"
|
||||
IMM="${DOCKERHUB_REPO}:${TAG}"
|
||||
CHN="${DOCKERHUB_REPO}:${NAME}"
|
||||
|
||||
echo "→ Building ${DF}"
|
||||
docker build --no-cache -t "${LI}" -f "${DF}" .
|
||||
|
||||
echo "→ Tag & push immutable → ${IMM}"
|
||||
docker tag "${LI}" "${IMM}"
|
||||
docker push "${IMM}"
|
||||
|
||||
echo "→ Tag & push channel → ${CHN}"
|
||||
docker tag "${IMM}" "${CHN}"
|
||||
docker push "${CHN}"
|
||||
@@ -0,0 +1,105 @@
|
||||
name: Poll llama.cpp & Trigger Build
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 */4 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
poll-and-trigger:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- id: fetch
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
REPO_URL="https://github.com/ggml-org/llama.cpp.git"
|
||||
DEFAULT_REF=$(git ls-remote --symref "$REPO_URL" HEAD | awk '/^ref:/ {print $2}')
|
||||
echo "📌 Default branch: ${DEFAULT_REF#refs/heads/}"
|
||||
LATEST_SHA=$(git ls-remote "$REPO_URL" "$DEFAULT_REF" | cut -f1)
|
||||
if [[ -z "$LATEST_SHA" ]]; then echo "❌ No SHA found"; exit 1; fi
|
||||
echo "✅ Latest SHA: $LATEST_SHA"
|
||||
echo "latest_sha=$LATEST_SHA" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: previous
|
||||
shell: bash
|
||||
env:
|
||||
GH_REPO: ${{ github.repository }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "🔎 Checking for prior artifact 'last-llama-sha'…"
|
||||
ART_ID=$(curl -fsSL -H "Authorization: Bearer $GH_TOKEN" \
|
||||
"https://api.github.com/repos/${GH_REPO}/actions/artifacts?per_page=100" \
|
||||
| jq -r '.artifacts | map(select(.name=="last-llama-sha" and .expired==false)) | sort_by(.created_at) | reverse | .[0].id // empty')
|
||||
if [[ -n "$ART_ID" ]]; then
|
||||
echo "📦 Found artifact id: $ART_ID (downloading)"
|
||||
curl -fsSL -H "Authorization: Bearer $GH_TOKEN" -L \
|
||||
"https://api.github.com/repos/${GH_REPO}/actions/artifacts/${ART_ID}/zip" -o artifact.zip
|
||||
unzip -l artifact.zip || true
|
||||
unzip -p artifact.zip last_commit_sha > last_commit_sha || true
|
||||
else
|
||||
echo "ℹ️ No prior artifact found"
|
||||
fi
|
||||
PREV_SHA=""
|
||||
if [[ -f last_commit_sha ]]; then PREV_SHA=$(cat last_commit_sha); fi
|
||||
echo "🕓 Previous SHA: $PREV_SHA"
|
||||
echo "previous_sha=$PREV_SHA" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- id: compare
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "🧮 Comparing SHAs…"
|
||||
echo "prev: ${{ steps.previous.outputs.previous_sha }}"
|
||||
echo "curr: ${{ steps.fetch.outputs.latest_sha }}"
|
||||
if [[ "${{ steps.fetch.outputs.latest_sha }}" != "${{ steps.previous.outputs.previous_sha }}" ]]; then
|
||||
echo "🔁 New commit detected"
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "✅ No change"
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Trigger build_and_publish.yml on main
|
||||
if: steps.compare.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
WF="build_and_publish.yml"
|
||||
REF="main"
|
||||
echo "🚀 Dispatching $WF on $REF…"
|
||||
CODE=$(curl -s -o /tmp/resp -w "%{http_code}" \
|
||||
-X POST \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "Authorization: Bearer $GH_TOKEN" \
|
||||
-d "{\"ref\":\"$REF\",\"inputs\":{\"backends\":\"all\"}}" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/actions/workflows/$WF/dispatches")
|
||||
echo "HTTP $CODE"
|
||||
if [[ "$CODE" != "204" ]]; then echo "Response:"; cat /tmp/resp; exit 1; fi
|
||||
|
||||
|
||||
- name: Save new SHA
|
||||
if: steps.compare.outputs.changed == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
printf "%s" "${{ steps.fetch.outputs.latest_sha }}" > last_commit_sha
|
||||
echo "💾 Saved $(wc -c < last_commit_sha) bytes to $(pwd)/last_commit_sha"
|
||||
ls -la last_commit_sha
|
||||
|
||||
- name: Upload last-SHA artifact
|
||||
if: steps.compare.outputs.changed == 'true'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: last-llama-sha
|
||||
path: last_commit_sha
|
||||
retention-days: 7
|
||||
@@ -0,0 +1,89 @@
|
||||
name: Prune Old Toolbox Images
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
backends:
|
||||
description: Comma-separated backends to prune (e.g. "rocm-7beta,rocm-7rc") or "all"
|
||||
default: all
|
||||
keep:
|
||||
description: Number of latest tags to keep
|
||||
default: "3"
|
||||
workflow_run:
|
||||
workflows: ["Build & Publish AMD Strix Halo Toolboxes"]
|
||||
types: [completed] # runs after success/failure/cancel
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
prune:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
NS: kyuz0
|
||||
REPO: amd-strix-halo-toolboxes
|
||||
steps:
|
||||
- name: Install jq
|
||||
run: sudo apt-get update && sudo apt-get install -y jq
|
||||
|
||||
- name: Login to Docker Hub API (JWT)
|
||||
id: login
|
||||
env:
|
||||
DH_USER: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
DH_PASS: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
run: |
|
||||
TOKEN=$(curl -s -H "Content-Type: application/json" \
|
||||
-d "{\"username\":\"${DH_USER}\",\"password\":\"${DH_PASS}\"}" \
|
||||
https://hub.docker.com/v2/users/login/ | jq -r .token)
|
||||
if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]; then
|
||||
echo "Failed to get Docker Hub JWT"; exit 1
|
||||
fi
|
||||
echo "token=${TOKEN}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Determine backend list
|
||||
id: mk
|
||||
shell: bash
|
||||
run: |
|
||||
IN='${{ github.event.inputs.backends }}'
|
||||
if [[ "$IN" == "all" || -z "$IN" ]]; then
|
||||
JSON='["rocm-6.4.2","rocm-6.4.2-rocwmma","rocm-6.4.3","rocm-6.4.3-rocwmma","rocm-6.4.4","rocm-6.4.4-rocwmma","rocm-7.1","rocm-7.1-rocwmma","rocm-7beta","rocm-7alpha","rocm-7alpha-rocwmma","rocm-7alpha-rocwmma-improved","rocm-7rc","rocm-7rc-rocwmma","rocm-7rc-rocwmma-fa_all_quants","vulkan-amdvlk","vulkan-radv"]'
|
||||
else
|
||||
IN_CLEAN=$(echo "$IN" | tr -d '[:space:]')
|
||||
JSON='["'${IN_CLEAN//,/\",\"}'"]'
|
||||
fi
|
||||
echo "list=${JSON}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Prune old tags
|
||||
env:
|
||||
TOKEN: ${{ steps.login.outputs.token }}
|
||||
KEEP: ${{ github.event.inputs.keep }}
|
||||
run: |
|
||||
BACKENDS='${{ steps.mk.outputs.list }}'
|
||||
mapfile -t ARR < <(jq -r '.[]' <<< "$BACKENDS")
|
||||
base_url="https://hub.docker.com/v2/repositories/${NS}/${REPO}/tags"
|
||||
auth_hdr="Authorization: JWT ${TOKEN}"
|
||||
|
||||
for B in "${ARR[@]}"; do
|
||||
echo ""
|
||||
echo "=== Backend: ${B} (keeping latest ${KEEP}) ==="
|
||||
next="${base_url}?page_size=100&ordering=last_updated&name=${B}_"
|
||||
tags=()
|
||||
while [[ -n "$next" && "$next" != "null" ]]; do
|
||||
resp=$(curl -s -H "$auth_hdr" "$next")
|
||||
page_tags=($(jq -r '.results[].name' <<< "$resp" | grep -E "^${B}_" || true))
|
||||
tags+=("${page_tags[@]}")
|
||||
next=$(jq -r '.next' <<< "$resp")
|
||||
done
|
||||
total=${#tags[@]}
|
||||
echo "Found ${total} immutable tag(s) for ${B}."
|
||||
if (( total <= KEEP )); then
|
||||
echo "Nothing to delete."
|
||||
continue
|
||||
fi
|
||||
to_delete=("${tags[@]:KEEP}")
|
||||
for t in "${to_delete[@]}"; do
|
||||
echo "Deleting tag ${t}..."
|
||||
curl -s -X DELETE -H "$auth_hdr" \
|
||||
"https://hub.docker.com/v2/repositories/${NS}/${REPO}/tags/${t}/" \
|
||||
-o /dev/null -w "%{http_code}\n" | grep -Eq "^(202|204)$" \
|
||||
&& echo "✔ Deleted" || echo "✖ Failed"
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1 @@
|
||||
__pycache__
|
||||
@@ -0,0 +1,351 @@
|
||||
# AMD R9700 Llama.cpp Toolboxes
|
||||
|
||||
This project provides pre-built containers (“toolboxes”) for running LLMs on **AMD Radeon AI PRO R9700** GPUs. Toolbx is the standard developer container system in Fedora (and now works on Ubuntu, openSUSE, Arch, etc).
|
||||
|
||||
## 🚨 Updates — 2025-11-18
|
||||
|
||||
- Released new toolboxes for ROCm 7 that track the nightly builds, these are now called `nightly`.
|
||||
- Updated and extended benchmakrs across all llama.cpp backend configurations, and included bennchmarks over RPC (two nodes) and long context (32k) -> [Interactive Benchmark Viewer](https://kyuz0.github.io/amd-r9700-toolboxes/)
|
||||
|
||||
## Watch the YouTube Video
|
||||
|
||||
[](https://youtu.be/wCBLMXgk3No)
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Quick Answers (Read This First)](#quick-answers-read-this-first)
|
||||
1. [Llama.cpp Compiled for Every Backend](#1-llamacpp-compiled-for-every-backend)
|
||||
1.1 [Supported Container Images](#11-supported-container-images)
|
||||
2. [Quickest Usage Example](#2-quickest-usage-example)
|
||||
2.1 [Creating the toolboxes with GPU access](#21-creating-the-toolboxes-with-gpu-access)
|
||||
2.2 [Running models inside the toolboxes](#22-running-models-inside-the-toolboxes)
|
||||
2.3 [Downloading GGUF Models from HuggingFace](#23-downloading-gguf-models-from-huggingface)
|
||||
3. [Performance Benchmarks](#3-performance-benchmarks)
|
||||
4. [Memory Planning & VRAM Estimator](#4-memory-planning--vram-estimator)
|
||||
5. [Building Containers Locally](#5-building-containers-locally)
|
||||
6. [Host Configuration](#6-host-configuration)
|
||||
6.1 [Test Configuration](#61-test-configuration)
|
||||
6.2 [Kernel Parameters (tested on Fedora 42)](#62-kernel-parameters-tested-on-fedora-42)
|
||||
6.3 [Ubuntu 24.04](#63-ubuntu-2404)
|
||||
7. [More Documentation](#7-more-documentation)
|
||||
8. [References](#8-references)
|
||||
|
||||
|
||||
## Quick Answers (Read This First)
|
||||
|
||||
### How do I get a toolbox up and running?
|
||||
|
||||
**Command — Create Vulkan (RADV) toolbox**
|
||||
|
||||
```sh
|
||||
toolbox create llama-vulkan-radv \
|
||||
--image docker.io/kyuz0/amd-r9700-toolboxes:vulkan-radv \
|
||||
-- --device /dev/dri --group-add video --security-opt seccomp=unconfined
|
||||
```
|
||||
|
||||
**Command — Create ROCm toolbox (6.4.4/7.1/7.9/7-nightly)**
|
||||
|
||||
```sh
|
||||
toolbox create llama-rocm-7.1-rocwmma \
|
||||
--image docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.1-rocwmma \
|
||||
-- --device /dev/dri --device /dev/kfd \
|
||||
--group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined
|
||||
```
|
||||
|
||||
**Command — Enter the toolbox shell**
|
||||
|
||||
```sh
|
||||
toolbox enter llama-vulkan-radv
|
||||
```
|
||||
|
||||
**Command — List detected GPUs (inside the toolbox)**
|
||||
|
||||
```sh
|
||||
llama-cli --list-devices
|
||||
```
|
||||
|
||||
### How do I download weights for a model?
|
||||
|
||||
**Command — Download a GGUF shard from Hugging Face**
|
||||
|
||||
```bash
|
||||
HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF \
|
||||
BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
--local-dir models/qwen3-coder-30B-A3B/
|
||||
```
|
||||
|
||||
`HF_HUB_ENABLE_HF_TRANSFER=1` turns on the Rust-based accelerated downloader (`pip install hf-transfer`).
|
||||
|
||||
### How do I run llama-server (and llama-cli) with a model?
|
||||
|
||||
Flash attention and no-memory-map **must** be enabled or R9700 will crawl/crash.
|
||||
|
||||
**Command — Run llama-server with flash attention + no-mmap**
|
||||
|
||||
```sh
|
||||
llama-server -m models/qwen3-coder-30B-A3B/BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
-c 8192 -ngl 999 -fa 1 --no-mmap
|
||||
```
|
||||
|
||||
**Command — Run llama-cli with the same essentials**
|
||||
|
||||
```sh
|
||||
llama-cli --no-mmap -ngl 999 -fa 1 -m models/qwen3-coder-30B-A3B/BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
-p "Write a R9700 toolkit haiku."
|
||||
```
|
||||
|
||||
### How do I keep the toolboxes updated?
|
||||
|
||||
**Command — Refresh every toolbox**
|
||||
|
||||
```bash
|
||||
./refresh-toolboxes.sh all
|
||||
```
|
||||
|
||||
**Command — Refresh specific toolboxes**
|
||||
|
||||
```bash
|
||||
./refresh-toolboxes.sh llama-vulkan-radv llama-rocm-7.1-rocwmma
|
||||
```
|
||||
|
||||
## 1. Llama.cpp Compiled for Every Backend
|
||||
|
||||
This project uses [Llama.cpp](https://github.com/ggerganov/llama.cpp), a high-performance inference engine for running local LLMs (large language models) on CPUs and GPUs. Llama.cpp is open source, extremely fast, and is the only engine supporting all key backends for AMD R9700: Vulkan (RADV, AMDVLK) and ROCm/HIP
|
||||
|
||||
* **Vulkan** is a cross-platform, low-level graphics and compute API. Llama.cpp can use Vulkan for GPU inference with either the open Mesa RADV driver or AMD's "official" open AMDVLK driver. This is the most stable and supported option for AMD CPUs at the moment.
|
||||
* **ROCm** is AMD's open-source answer to CUDA: a GPU compute stack for machine learning and HPC. With ROCm, you can run Llama.cpp on AMD GPUs in a way similar to how CUDA works on NVIDIA - this is not the most stable/mature, but recently it's been getting better.
|
||||
|
||||
### 1.1 Supported Container Images
|
||||
|
||||
You can check the containers on DockerHub: https://hub.docker.com/r/kyuz0/amd-r9700-toolboxes/tags.
|
||||
|
||||
| Container Tag | Backend/Stack | Purpose / Notes |
|
||||
| ------------------------------ | -------------------------------------- | --------------- |
|
||||
| `vulkan-amdvlk` | Vulkan (AMDVLK) | Fastest backend—AMD open-source driver. ≤2 GiB single buffer allocation limit, some large models won't load. |
|
||||
| `vulkan-radv` | Vulkan (Mesa RADV) | Most stable and compatible. Recommended for most users and all models. |
|
||||
| `rocm-6.4.4` | ROCm 6.4.4 (HIP) + hipBLASLt* | Latest stable build for ROCm 6.4.4, performs very well with most model architectures/quants. |
|
||||
| `rocm-6.4.4-rocwmma` | ROCm 6.4.4 + ROCWMMA + hipBLASLt* | 6.4.4 with ROCWMMA enabled for better flash attention on RDNA3+/CDNA. |
|
||||
| `rocm-7.1` | ROCm 7.1 GA (HIP) + hipBLASLt* | Current GA release for ROCm 7.x; improved scheduler and hipBLASLt kernels. |
|
||||
| `rocm-7.1-rocwmma` | ROCm 7.1 GA + ROCWMMA + hipBLASLt* | 7.1 with ROCWMMA for maximum flash-attention throughput. |
|
||||
| `rocm-7.9` | ROCm 7.9 (HIP) + hipBLASLt* | Used to be the release candidate for ROCm 7.9.0 (hence the `rc` tag in the name), now released. |
|
||||
| `rocm-7.9-rocwmma` | ROCm 7.9 + ROCWMMA + hipBLASLt* | 7.9.0 build with ROCWMMA—useful for early flash-attention validation. |
|
||||
| `rocm-7-nightly` | ROCm 7 Nightly (“7rc-alpha”) + hipBLASLt* | Tracks ROCm 7 nightly (alpha) preview with bleeding-edge patches. |
|
||||
| `rocm-7-nightly-rocwmma` | ROCm 7 Nightly + ROCWMMA + hipBLASLt* | Same nightly/alpha stack with ROCWMMA tuned for flash attention. |
|
||||
|
||||
\* All these toolboxes export `ROCBLAS_USE_HIPBLASLT=1` because it historically delivered better performance and stability, altough this might not be the case any more.
|
||||
|
||||
> These containers are **automatically** rebuilt whenever the Llama.cpp master branch is updated, ensuring you get the latest bug fixes and new model support. The easiest way to update to the newest versions is by running the `refresh-toolboxes.sh` [script below](#211-toolbox-refresh-script-automatic-updates).
|
||||
>
|
||||
> Legacy images `rocm-6.4.2` and `rocm-6.4.3` are still on Docker Hub for reproducibility but are intentionally excluded from the active list above. Prefer `rocm-6.4.4+` or any `rocm-7.x` tag unless you must bisect an old regression.
|
||||
|
||||
---
|
||||
|
||||
## 2. Quickest Usage Example
|
||||
|
||||
### 2.1 Creating the toolboxes with GPU access
|
||||
|
||||
To use Llama.cpp with hardware acceleration inside a toolbox container, you must expose the right GPU device nodes from your host. The exact flags depend on the backend.
|
||||
|
||||
#### Command — Create Vulkan (RADV/AMDVLK) toolbox
|
||||
|
||||
```sh
|
||||
toolbox create llama-vulkan-radv \
|
||||
--image docker.io/kyuz0/amd-r9700-toolboxes:vulkan-radv \
|
||||
-- --device /dev/dri --group-add video --security-opt seccomp=unconfined
|
||||
```
|
||||
|
||||
*Only `/dev/dri` is required for Vulkan. Make sure your user is in the `video` group.*
|
||||
|
||||
#### Command — Create ROCm toolbox (swap the tag for 6.4.4, 7.1, 7.9, 7-nightly…)
|
||||
|
||||
```sh
|
||||
toolbox create llama-rocm-7.1-rocwmma \
|
||||
--image docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.1-rocwmma \
|
||||
-- --device /dev/dri --device /dev/kfd \
|
||||
--group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined
|
||||
```
|
||||
|
||||
*ROCm needs both `/dev/dri` and `/dev/kfd`, plus the `video`, `render`, and sometimes `sudo` groups for full compute access. Swap `rocm-7.1-rocwmma` for any other active ROCm tag (6.4.4, 7.9, 7-nightly, etc.).*
|
||||
|
||||
> **Note:**
|
||||
>
|
||||
> * `--device /dev/dri` provides graphics/video device nodes.
|
||||
> * `--device /dev/kfd` is required for ROCm compute.
|
||||
> * Extra groups (`video`, `render`, `sudo`) may be required for full access to GPU nodes and compute features, especially with ROCm.
|
||||
> * Use `--security-opt seccomp=unconfined` to avoid seccomp sandbox issues (needed for some GPU syscalls).
|
||||
|
||||
### 2.1.1 Ubuntu users
|
||||
|
||||
Ubuntu’s `toolbox` package still breaks GPU access, so follow gyhor’s [issue comment](https://github.com/kyuz0/amd-r9700-toolboxes/issues/16#issuecomment-3582028864) and use [Distrobox](https://github.com/89luca89/distrobox) instead:
|
||||
|
||||
```sh
|
||||
distrobox create -n llama-rocm-7.1 \
|
||||
--image docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.1-rocwmma \
|
||||
--additional-flags "--device /dev/kfd --device /dev/dri --group-add video --group-add render --security-opt seccomp=unconfined"
|
||||
distrobox enter llama-rocm-7.1
|
||||
llama-cli --list-devices
|
||||
```
|
||||
|
||||
### 2.1.2 Toolbox Refresh Script (Automatic Updates)
|
||||
|
||||
To pull the latest container images and recreate toolboxes cleanly, use the provided script:
|
||||
|
||||
#### 📦 `refresh-toolboxes.sh`
|
||||
|
||||
```bash
|
||||
./refresh-toolboxes.sh all
|
||||
```
|
||||
|
||||
This will:
|
||||
|
||||
1. Delete existing toolboxes (if any)
|
||||
2. Pull the latest images from DockerHub
|
||||
3. Recreate each toolbox with correct GPU access flags
|
||||
|
||||
You can also refresh just one or more toolboxes:
|
||||
|
||||
```bash
|
||||
./refresh-toolboxes.sh llama-vulkan-radv llama-rocm-7.1-rocwmma
|
||||
```
|
||||
|
||||
### 2.2 Running models inside the toolboxes
|
||||
|
||||
#### Command — Enter the toolbox shell
|
||||
|
||||
```sh
|
||||
toolbox enter llama-vulkan-radv
|
||||
```
|
||||
|
||||
*This drops you into a shell inside the toolbox using your regular user account. The container shares your host home directory—anything in `$HOME` is accessible and writable inside the toolbox, so treat it like your host shell.*
|
||||
|
||||
#### Command — Confirm Llama.cpp sees your GPU
|
||||
|
||||
```sh
|
||||
llama-cli --list-devices
|
||||
```
|
||||
|
||||
Run this inside the toolbox to verify RADV/AMDVLK/ROCm devices are visible before loading a multi-gigabyte model.
|
||||
|
||||
> ⚠️ Always pass **flash attention** and **no-memory-map** flags when running on R9700. `llama-server` and `llama-cli` both expect `-fa 1 --no-mmap`. Skipping either tanks performance or triggers kernel crashes because of the giant unified memory aperture.
|
||||
|
||||
#### Command — Run llama-cli with flash attention + no-mmap
|
||||
|
||||
```sh
|
||||
llama-cli --no-mmap -ngl 999 -fa 1 \
|
||||
-m models/qwen3-coder-30B-A3B/BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
-p "Write a R9700 toolkit haiku."
|
||||
```
|
||||
|
||||
- `-ngl 999` forces every layer onto the GPU.
|
||||
- `-fa 1` turns on flash attention; omit it and throughput collapses.
|
||||
- `--no-mmap` keeps allocations in unified memory rather than trying to memory-map multi-gigabyte files.
|
||||
|
||||
#### Command — Run llama-server with flash attention + no-mmap
|
||||
|
||||
```sh
|
||||
llama-server -m models/qwen3-coder-30B-A3B/BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
-c 8192 -ngl 999 -fa 1 --no-mmap
|
||||
```
|
||||
|
||||
Adjust `-c` for context length and never drop `-fa 1 --no-mmap`.
|
||||
|
||||
## 2.3 Downloading GGUF Models from HuggingFace
|
||||
|
||||
Most Llama.cpp-compatible models are on [HuggingFace](https://huggingface.co/models?format=gguf). Filter for **GGUF** format, and try to pick Unsloth quantizations—they work great and are actively updated: https://huggingface.co/unsloth.
|
||||
|
||||
Download using the Hugging Face CLI. For example, to get the first shard of Qwen3 Coder 30B BF16 (https://huggingface.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF):
|
||||
|
||||
#### Command — Download a GGUF shard with `huggingface-cli`
|
||||
|
||||
```bash
|
||||
HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF \
|
||||
BF16/Qwen3-Coder-30B-A3B-Instruct-BF16-00001-of-00002.gguf \
|
||||
--local-dir models/qwen3-coder-30B-A3B/
|
||||
```
|
||||
|
||||
`HF_HUB_ENABLE_HF_TRANSFER=1` uses a Rust-based package that enables faster download (install from [Pypi](https://pypi.org/project/hf-transfer/)).
|
||||
|
||||
## 3. Performance Benchmarks
|
||||
|
||||
🌐 Interactive exploration of the latest benchmark runs: [Interactie Benchmark Viewer](https://kyuz0.github.io/amd-r9700-toolboxes/)
|
||||
|
||||
## 4. Memory Planning & VRAM Estimator
|
||||
|
||||
Running large language models locally requires estimating **total VRAM required**—not just for the model weights, but also for the "context" (number of active tokens) and extra overhead.
|
||||
|
||||
Use `gguf-vram-estimator.py` to check exactly how much memory you need for a given `.gguf` model and target context length. Example output:
|
||||
|
||||
```
|
||||
$ gguf-vram-estimator.py models/llama-4-scout-17b-16e/Q4_K_XL/Llama-4-Scout-17B-16E-Instruct-UD-Q4_K_XL-00001-of-00002.gguf --contexts 4096 32768 1048576
|
||||
|
||||
--- Model 'Llama-4-Scout-17B-16E-Instruct' ---
|
||||
Max Context: 10,485,760 tokens
|
||||
Model Size: 57.74 GiB
|
||||
Incl. Overhead: 2.00 GiB
|
||||
|
||||
--- Memory Footprint Estimation ---
|
||||
Context Size | Context Memory | Est. Total VRAM
|
||||
---------------------------------------------------
|
||||
4,096 | 1.88 GiB | 61.62 GiB
|
||||
32,768 | 15.06 GiB | 74.80 GiB
|
||||
1,048,576 | 49.12 GiB | 108.87 GiB
|
||||
```
|
||||
|
||||
With Q4\_K quantization, **Llama-4-Scout 17B** can reach a 1M token context and still fit within a 128GB system, but... **it will be extremely slow to process such a long context**: see benchmarks (e.g. \~200 tokens/sec for prompt processing). Processing a 1M token context may take hours.
|
||||
|
||||
Contrast: Qwen3-235B Q3\_K (quantized, 97GiB model):
|
||||
|
||||
```
|
||||
$ gguf-vram-estimator.py models/qwen3-235B-Q3_K-XL/UD-Q3_K_XL/Qwen3-235B-A22B-Instruct-2507-UD-Q3_K_XL-00001-of-00003.gguf --contexts 65536 131072 262144
|
||||
|
||||
--- Memory Footprint Estimation ---
|
||||
Context Size | Context Memory | Est. Total VRAM
|
||||
---------------------------------------------------
|
||||
65,536 | 11.75 GiB | 110.75 GiB
|
||||
131,072 | 23.50 GiB | 122.50 GiB
|
||||
262,144 | 47.00 GiB | 146.00 GiB
|
||||
```
|
||||
|
||||
For Qwen3-235B, **128GB RAM allows you to run with context up to \~130k tokens.**
|
||||
|
||||
* The estimator lets you plan ahead and avoid out-of-memory errors when loading or using models.
|
||||
* For more examples and a breakdown of VRAM components, see [docs/vram-estimator.md](docs/vram-estimator.md).
|
||||
|
||||
---
|
||||
|
||||
## 5. Building Containers Locally
|
||||
|
||||
Pre-built toolbox container images are published on Docker Hub for immediate use. If you wish to build the containers yourself (for example, to customize packages or rebuild with a different llama.cpp version), see:
|
||||
|
||||
Full instructions: [docs/building.md](docs/building.md).
|
||||
|
||||
---
|
||||
|
||||
## 6. Host Configuration
|
||||
|
||||
This should work on any R9700. For a complete list of available hardware, see: [R9700 Hardware Database](https://r9700-homelab.d7.wtf/Hardware)
|
||||
|
||||
### 6.1 Test Configuration
|
||||
|
||||
| | |
|
||||
| ----------------- | --------------------------------------------- |
|
||||
| **Test Machine** | HP Z2 Mini G1a |
|
||||
| **CPU** | AMD Radeon AI PRO R9700 |
|
||||
| **System Memory** | 64 GB RAM (Host) |
|
||||
| **GPU Memory** | 32 GB GDDR6 |
|
||||
| **Host OS** | Fedora 42, kernel 6.15.6-200.fc42.x86\_86\_64 |
|
||||
|
||||
|
||||
|
||||
## 7. More Documentation
|
||||
|
||||
* [docs/benchmarks.md](docs/benchmarks.md): Full benchmark logs, model list, parsed results
|
||||
* [docs/vram-estimator.md](docs/vram-estimator.md): Memory planning, practical example runs
|
||||
* [docs/building.md](docs/building.md): Local build, toolbox customization, advanced use
|
||||
|
||||
## 8. References
|
||||
|
||||
* The main reference for AMD R9700 home labs, by deseven (there's also a Discord server): [https://r9700-homelab.d7.wtf/](https://r9700-homelab.d7.wtf/)
|
||||
* Most comprehesive repostiry of test builds for R9700 by lhl -> [https://github.com/lhl/r9700-testing/tree/main](https://github.com/lhl/strix-halo-testing/tree/main)
|
||||
* Ubuntu 24.04 configuration
|
||||
[https://github.com/technigmaai/technigmaai-wiki/wiki/AMD-Ryzen-AI-Max--395:-GTT--Memory-Step%E2%80%90by%E2%80%90Step-Instructions-(Ubuntu-24.04)](https://github.com/technigmaai/technigmaai-wiki/wiki/AMD-Ryzen-AI-Max--395:-GTT--Memory-Step%E2%80%90by%E2%80%90Step-Instructions-%28Ubuntu-24.04%29)
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
RESULTS_DIR_DEFAULT = "results"
|
||||
|
||||
# Same detection logic as your extractor
|
||||
HEADER_RE = re.compile(r"^\|\s*model\s*\|", re.IGNORECASE)
|
||||
SEP_RE = re.compile(r"^\|\s*-+")
|
||||
|
||||
LOAD_ERR = re.compile(r"failed to load model|Device memory allocation.*failed|⚠️\s*Fail", re.IGNORECASE)
|
||||
HANG_ERR = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
|
||||
GENERIC_ERR = re.compile(r"error:|exit \d+|runtime error|⚠️\s*Runtime Error", re.IGNORECASE)
|
||||
|
||||
|
||||
def parse_table(text):
|
||||
lines = text.splitlines()
|
||||
rows = []
|
||||
header = None
|
||||
col_idx = {}
|
||||
|
||||
for line in lines:
|
||||
if HEADER_RE.search(line):
|
||||
header = [c.strip().lower() for c in line.strip().strip("|").split("|")]
|
||||
for idx, name in enumerate(header):
|
||||
col_idx[name] = idx
|
||||
continue
|
||||
|
||||
if header and (SEP_RE.search(line) or not line.strip()):
|
||||
continue
|
||||
|
||||
if header and line.startswith("|"):
|
||||
parts = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if len(parts) < len(header):
|
||||
continue
|
||||
row = {}
|
||||
for name, idx in col_idx.items():
|
||||
row[name] = parts[idx]
|
||||
rows.append(row)
|
||||
|
||||
if header and line.strip() == "" and rows:
|
||||
break
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def detect_error(text):
|
||||
if LOAD_ERR.search(text):
|
||||
return True
|
||||
if HANG_ERR.search(text):
|
||||
return True
|
||||
if GENERIC_ERR.search(text):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_non_transient_vram_issue(text):
|
||||
# Do NOT delete logs with this kind of Vulkan OOM
|
||||
return (
|
||||
"ggml_vulkan: Device memory allocation of size" in text
|
||||
and "Requested buffer size exceeds device buffer size limit" in text
|
||||
)
|
||||
|
||||
|
||||
def is_failed_run(text):
|
||||
table_rows = parse_table(text)
|
||||
|
||||
has_pp = any(r.get("test", "").lower() == "pp512" for r in table_rows)
|
||||
has_tg = any(r.get("test", "").lower() == "tg128" for r in table_rows)
|
||||
|
||||
if has_pp or has_tg:
|
||||
return False
|
||||
|
||||
return detect_error(text)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Delete transient-failure benchmark logs in results/"
|
||||
)
|
||||
ap.add_argument(
|
||||
"--results-dir",
|
||||
default=RESULTS_DIR_DEFAULT,
|
||||
help="Directory containing *.log files (default: results)",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Only print what would be deleted",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
results_dir = args.results_dir
|
||||
pattern = os.path.join(results_dir, "*.log")
|
||||
|
||||
to_delete = []
|
||||
skipped_non_transient = []
|
||||
|
||||
for path in sorted(glob.glob(pattern)):
|
||||
try:
|
||||
with open(path, errors="ignore") as f:
|
||||
text = f.read()
|
||||
except OSError as e:
|
||||
print(f"Could not read {path}: {e}")
|
||||
continue
|
||||
|
||||
if not is_failed_run(text):
|
||||
continue
|
||||
|
||||
if is_non_transient_vram_issue(text):
|
||||
skipped_non_transient.append(path)
|
||||
continue
|
||||
|
||||
to_delete.append(path)
|
||||
|
||||
if not to_delete and not skipped_non_transient:
|
||||
print("No failed logs found.")
|
||||
return
|
||||
|
||||
if skipped_non_transient:
|
||||
print("Keeping logs with non transient VRAM issues:")
|
||||
for p in skipped_non_transient:
|
||||
print(f" KEEP {p}")
|
||||
|
||||
if to_delete:
|
||||
print("Deleting logs with transient failures:")
|
||||
for p in to_delete:
|
||||
print(f" DELETE {p}")
|
||||
if not args.dry_run:
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError as e:
|
||||
print(f" Failed to delete {p}: {e}")
|
||||
else:
|
||||
print("No logs to delete.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,571 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
gen_benchmarks_md.py — Generate Markdown for README + detailed benchmarks from results.json
|
||||
|
||||
Defaults:
|
||||
- Input JSON: ../docs/results.json
|
||||
- Outputs: ./README_benchmarks_section.md and ./benchmarks_generated.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import argparse
|
||||
import statistics as stats
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
|
||||
# === ENV LABELS ===
|
||||
ENV_LABEL: Dict[str, str] = {
|
||||
# ROCm 7 RC
|
||||
"rocm7_rc-rocwmma": "ROCm 7 RC + ROCWMMA + hipBLASLt",
|
||||
"rocm7_rc": "ROCm 7 RC (hipBLASLt)",
|
||||
"rocm7_rc-hblt0": "ROCm 7 RC (hipBLASLt OFF)",
|
||||
"rocm7_rc-rocwmma-hblt0": "ROCm 7 RC + ROCWMMA (hipBLASLt OFF)",
|
||||
|
||||
# ROCm 6.4.4
|
||||
"rocm6_4_4": "ROCm 6.4.4 (hipBLASLt)",
|
||||
"rocm6_4_4-hblt0": "ROCm 6.4.4 (hipBLASLt OFF)",
|
||||
"rocm6_4_4-rocwmma": "ROCm 6.4.4 + ROCWMMA (hipBLASLt)",
|
||||
"rocm6_4_4-rocwmma-hblt0": "ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF)",
|
||||
|
||||
# Vulkan
|
||||
"vulkan_amdvlk": "Vulkan AMDVLK",
|
||||
"vulkan_radv": "Vulkan RADV",
|
||||
}
|
||||
|
||||
TESTS = ["pp512", "tg128"]
|
||||
|
||||
def md_row(values: List[str]) -> str:
|
||||
return "| " + " | ".join(values) + " |"
|
||||
|
||||
|
||||
def load_results(path: Path) -> Dict:
|
||||
data = json.loads(path.read_text())
|
||||
assert "runs" in data and isinstance(data["runs"], list), "results.json must have a top-level 'runs' list"
|
||||
return data
|
||||
|
||||
|
||||
def envs_present(runs: List[Dict], only_env: Optional[List[str]], include_all_envs: bool) -> List[str]:
|
||||
present = {r.get("env") for r in runs if r.get("env")}
|
||||
if only_env:
|
||||
present = present.intersection(set(only_env))
|
||||
if include_all_envs:
|
||||
# Include even if not present (might appear 0 rows in tables)
|
||||
envs = [e for e in ENV_LABEL.keys() if (not only_env or e in only_env)]
|
||||
else:
|
||||
envs = [e for e in ENV_LABEL.keys() if e in present and (not only_env or e in only_env)]
|
||||
return envs
|
||||
|
||||
|
||||
def fa_to_filter(fa: str) -> Optional[bool]:
|
||||
fa = fa.lower().strip()
|
||||
if fa == "on":
|
||||
return True
|
||||
if fa == "off":
|
||||
return False
|
||||
if fa == "any":
|
||||
return None
|
||||
raise ValueError("--fa must be on/off/any")
|
||||
|
||||
|
||||
def margin_aware_placements(
|
||||
runs: List[Dict],
|
||||
envs: List[str],
|
||||
test_filter: str,
|
||||
fa_filter: Optional[bool]
|
||||
) -> Tuple[Dict[str, Dict[str, int]], int]:
|
||||
"""
|
||||
Returns (placements, sample_count)
|
||||
placements[env] -> {"first": n, "second": n, "third": n}
|
||||
sample_count = number of model+quant comparisons considered
|
||||
"""
|
||||
placements = defaultdict(lambda: {"first": 0, "second": 0, "third": 0})
|
||||
# group by (model, quant)
|
||||
grouped = defaultdict(list)
|
||||
for r in runs:
|
||||
if r.get("error"):
|
||||
continue
|
||||
if r.get("test") != test_filter:
|
||||
continue
|
||||
if fa_filter is not None and r.get("fa") != fa_filter:
|
||||
continue
|
||||
if r.get("env") not in envs:
|
||||
continue
|
||||
key = (r.get("model_clean"), r.get("quant"))
|
||||
grouped[key].append(r)
|
||||
|
||||
samples = 0
|
||||
for key, entries in grouped.items():
|
||||
# collate by env
|
||||
env_groups = defaultdict(list)
|
||||
for e in entries:
|
||||
env_groups[e["env"]].append(e)
|
||||
env_list = [e for e in envs if e in env_groups] # keep requested order
|
||||
if len(env_list) < 2:
|
||||
continue
|
||||
|
||||
# summarize median mean ± median err per env
|
||||
summary = {}
|
||||
for env in env_list:
|
||||
means = [x["tps_mean"] for x in env_groups[env] if x.get("tps_mean") is not None]
|
||||
errs = [x.get("tps_err", 0.0) or 0.0 for x in env_groups[env]]
|
||||
if not means:
|
||||
continue
|
||||
m = stats.median(means)
|
||||
e = stats.median(errs) if errs else 0.0
|
||||
summary[env] = (m - e, m + e, m)
|
||||
if len(summary) < 2:
|
||||
continue
|
||||
|
||||
samples += 1
|
||||
|
||||
# rank with overlap -> ties share rank
|
||||
remaining = [env for env, _ in sorted(summary.items(), key=lambda kv: kv[1][2], reverse=True)]
|
||||
assigned = {}
|
||||
current_rank = 1
|
||||
while remaining and current_rank <= 3:
|
||||
env0 = remaining[0]
|
||||
low0, high0, _ = summary[env0]
|
||||
tied = [env0]
|
||||
for env in remaining[1:]:
|
||||
low, high, _ = summary[env]
|
||||
if not (low > high0 or high < low0): # overlap -> tie
|
||||
tied.append(env)
|
||||
for env in tied:
|
||||
assigned[env] = current_rank
|
||||
remaining = [e for e in remaining if e not in tied]
|
||||
current_rank += 1
|
||||
|
||||
for env, rk in assigned.items():
|
||||
if rk == 1:
|
||||
placements[env]["first"] += 1
|
||||
elif rk == 2:
|
||||
placements[env]["second"] += 1
|
||||
elif rk == 3:
|
||||
placements[env]["third"] += 1
|
||||
|
||||
return placements, samples
|
||||
|
||||
|
||||
def pairwise_win_counts(runs: List[Dict], envA: str, envB: str, test: str, fa_filter: Optional[bool]) -> Tuple[int, int, int, int]:
|
||||
A = {}
|
||||
B = {}
|
||||
for r in runs:
|
||||
if r.get("error") or r.get("test") != test:
|
||||
continue
|
||||
if fa_filter is not None and r.get("fa") != fa_filter:
|
||||
continue
|
||||
key = (r.get("model_clean"), r.get("quant"))
|
||||
if r.get("env") == envA:
|
||||
A[key] = r["tps_mean"]
|
||||
elif r.get("env") == envB:
|
||||
B[key] = r["tps_mean"]
|
||||
winsA = winsB = ties = 0
|
||||
for k in (set(A) & set(B)):
|
||||
if A[k] > B[k]:
|
||||
winsA += 1
|
||||
elif B[k] > A[k]:
|
||||
winsB += 1
|
||||
else:
|
||||
ties += 1
|
||||
total = winsA + winsB + ties
|
||||
return winsA, winsB, ties, total
|
||||
|
||||
|
||||
def average_ranks(place_dict: Dict[str, Dict[str, int]]) -> Dict[str, Optional[float]]:
|
||||
avg = {}
|
||||
for env, c in place_dict.items():
|
||||
total = c.get("first", 0) + c.get("second", 0) + c.get("third", 0)
|
||||
if total == 0:
|
||||
avg[env] = None
|
||||
else:
|
||||
avg[env] = round((1 * c.get("first", 0) + 2 * c.get("second", 0) + 3 * c.get("third", 0)) / total, 2)
|
||||
return avg
|
||||
|
||||
|
||||
def flash_attention_effect(runs: List[Dict], envs: List[str]) -> Dict[str, Dict[str, Dict[str, float]]]:
|
||||
"""
|
||||
Returns: effects[env][test] = {n_pairs, median_pct, min, max}
|
||||
Based on paired model+quant runs (ON vs OFF).
|
||||
"""
|
||||
model_pairs = defaultdict(lambda: defaultdict(dict)) # (env,test)->(model,quant)->{fa: tps}
|
||||
for r in runs:
|
||||
if r.get("error") or r.get("tps_mean") is None:
|
||||
continue
|
||||
if r.get("test") not in TESTS:
|
||||
continue
|
||||
if r.get("env") not in envs:
|
||||
continue
|
||||
model_key = (r.get("model_clean"), r.get("quant"))
|
||||
model_pairs[(r["env"], r["test"])][model_key][r.get("fa")] = r["tps_mean"]
|
||||
|
||||
summary = defaultdict(dict)
|
||||
for (env, test), d in model_pairs.items():
|
||||
deltas = []
|
||||
for mk, vals in d.items():
|
||||
if True in vals and False in vals and vals[False] > 0:
|
||||
deltas.append((vals[True] - vals[False]) / vals[False] * 100.0)
|
||||
if deltas:
|
||||
summary[env][test] = {
|
||||
"n_pairs": len(deltas),
|
||||
"median_pct": round(stats.median(deltas), 1),
|
||||
"min": round(min(deltas), 1),
|
||||
"max": round(max(deltas), 1),
|
||||
}
|
||||
return summary
|
||||
|
||||
|
||||
def rocwmma_effect(runs: List[Dict], pairs_to_compare: List[Tuple[str, str, str]], tests: List[str]) -> List[Tuple[str, str, str, str, int, float]]:
|
||||
"""
|
||||
Compare ROCWMMA ON vs OFF with same hipBLASLt state.
|
||||
Returns rows of (context_label, test, env_on, env_off, n_pairs, median_delta_pct)
|
||||
where delta_pct = median(ON/OFF - 1)*100 over common model+quant.
|
||||
"""
|
||||
rows = []
|
||||
for env_on, env_off, label in pairs_to_compare:
|
||||
for test in tests:
|
||||
data_on = defaultdict(list)
|
||||
data_off = defaultdict(list)
|
||||
for r in runs:
|
||||
if r.get("error") or r.get("test") != test:
|
||||
continue
|
||||
if r.get("env") == env_on:
|
||||
data_on[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
|
||||
elif r.get("env") == env_off:
|
||||
data_off[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
|
||||
common = sorted(set(data_on) & set(data_off))
|
||||
if not common:
|
||||
continue
|
||||
ratios = []
|
||||
for k in common:
|
||||
aon = stats.median(data_on[k])
|
||||
aoff = stats.median(data_off[k])
|
||||
if aoff > 0:
|
||||
ratios.append(aon / aoff - 1.0)
|
||||
if ratios:
|
||||
rows.append((label, test, env_on, env_off, len(ratios), round(100 * stats.median(ratios), 1)))
|
||||
return rows
|
||||
|
||||
|
||||
def hipblaslt_effect(runs: List[Dict], pairs_to_compare: List[Tuple[str, str, str]], tests: List[str]) -> List[Tuple[str, str, str, str, int, float]]:
|
||||
"""
|
||||
Compare hipBLASLt ON vs OFF with same ROCWMMA state.
|
||||
Returns rows of (context_label, test, env_on, env_off, n_pairs, median_delta_pct)
|
||||
where delta_pct = median(ON/OFF - 1)*100 over common model+quant.
|
||||
"""
|
||||
rows = []
|
||||
for env_on, env_off, label in pairs_to_compare:
|
||||
for test in tests:
|
||||
data_on = defaultdict(list)
|
||||
data_off = defaultdict(list)
|
||||
for r in runs:
|
||||
if r.get("error") or r.get("test") != test:
|
||||
continue
|
||||
if r.get("env") == env_on:
|
||||
data_on[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
|
||||
elif r.get("env") == env_off:
|
||||
data_off[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
|
||||
common = sorted(set(data_on) & set(data_off))
|
||||
if not common:
|
||||
continue
|
||||
ratios = []
|
||||
for k in common:
|
||||
aon = stats.median(data_on[k])
|
||||
aoff = stats.median(data_off[k])
|
||||
if aoff > 0:
|
||||
ratios.append(aon / aoff - 1.0)
|
||||
if ratios:
|
||||
rows.append((label, test, env_on, env_off, len(ratios), round(100 * stats.median(ratios), 1)))
|
||||
return rows
|
||||
|
||||
|
||||
def amdvlk_vs_radv(runs: List[Dict], fa_filter: Optional[bool]) -> List[Tuple[str, int, int, int, int]]:
|
||||
rows = []
|
||||
for test in TESTS:
|
||||
wa, wr, ties, total = pairwise_win_counts(runs, "vulkan_amdvlk", "vulkan_radv", test, fa_filter)
|
||||
rows.append((test, wa, wr, ties, total))
|
||||
return rows
|
||||
|
||||
|
||||
def winners(place_dict: Dict[str, Dict[str, int]], slot="first") -> Tuple[List[str], int]:
|
||||
max_count = max((c.get(slot, 0) for c in place_dict.values()), default=0)
|
||||
win_list = [env for env, c in place_dict.items() if c.get(slot, 0) == max_count and max_count > 0]
|
||||
return win_list, max_count
|
||||
|
||||
|
||||
def human_list(envs: List[str]) -> str:
|
||||
return ", ".join(ENV_LABEL.get(e, e) for e in envs) if envs else "—"
|
||||
|
||||
|
||||
def build_readme_section(
|
||||
envs: List[str],
|
||||
pp_place: Dict[str, Dict[str, int]],
|
||||
tg_place: Dict[str, Dict[str, int]],
|
||||
fa_filter: Optional[bool]
|
||||
) -> str:
|
||||
# Winners
|
||||
pp_wins, _ = winners(pp_place, "first")
|
||||
tg_wins, _ = winners(tg_place, "first")
|
||||
|
||||
lines: List[str] = []
|
||||
lines.append("## 3. Performance Benchmarks (Key Results)")
|
||||
lines.append("")
|
||||
lines.append("🌐 Interactive exploration of the latest benchmark runs: [Interactie Benchmark Viewer](https://kyuz0.github.io/amd-strix-halo-toolboxes/)")
|
||||
lines.append("")
|
||||
lines.append("Benchmarks were analysed with **error-aware ties** (mean ± σ). If two backends overlap within margins, they are treated as a tie. All placement counts below use **Flash Attention ON**.")
|
||||
lines.append("")
|
||||
|
||||
# Placement tables
|
||||
def place_table(title: str, place_dict: Dict[str, Dict[str, int]]):
|
||||
lines.append(f"**{title}**")
|
||||
lines.append(md_row(["Backend", "1st", "2nd", "3rd"]))
|
||||
lines.append(md_row(["---", "---:", "---:", "---:"]))
|
||||
order = sorted(place_dict.items(), key=lambda kv: (-kv[1].get("first", 0), -kv[1].get("second", 0), kv[0]))
|
||||
for env, c in order:
|
||||
lines.append(md_row([ENV_LABEL.get(env, env), str(c.get("first", 0)), str(c.get("second", 0)), str(c.get("third", 0))]))
|
||||
lines.append("")
|
||||
|
||||
place_table("Prompt Processing (pp512)", pp_place)
|
||||
place_table("Token Generation (tg128)", tg_place)
|
||||
|
||||
# Data-driven recommendations
|
||||
def total_score(c: Dict[str, int]) -> int:
|
||||
# weight 1st more than 2nd
|
||||
return c.get("first", 0) * 2 + c.get("second", 0)
|
||||
|
||||
best_bal_score = -1
|
||||
balanced: List[str] = []
|
||||
for env in envs:
|
||||
score = total_score(pp_place.get(env, {})) + total_score(tg_place.get(env, {}))
|
||||
if score > best_bal_score:
|
||||
best_bal_score = score
|
||||
balanced = [env]
|
||||
elif score == best_bal_score:
|
||||
balanced.append(env)
|
||||
|
||||
lines.append("### Summary & Recommendations")
|
||||
lines.append(f"- **Fastest prompt processing:** {human_list(pp_wins)} (most 1st-place finishes).")
|
||||
lines.append(f"- **Fastest token generation:** {human_list(tg_wins)} (most 1st-place finishes).")
|
||||
lines.append(f"- **Balanced choice:** {human_list(balanced)} (consistently near the top across PP/TG).")
|
||||
lines.append("")
|
||||
lines.append("> **Note (ROCm 7):** Toolboxes enable **hipBLASLt** by default. The benchmark suite also runs **hipBLASLt OFF** variants to show its impact.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_benchmarks_doc(
|
||||
runs: List[Dict],
|
||||
envs: List[str],
|
||||
pp_place: Dict[str, Dict[str, int]],
|
||||
tg_place: Dict[str, Dict[str, int]],
|
||||
fa_filter: Optional[bool],
|
||||
) -> str:
|
||||
lines: List[str] = []
|
||||
lines.append("# AMD Strix Halo — llama.cpp Toolboxes (Benchmarks)")
|
||||
lines.append("")
|
||||
lines.append("**Interactive results:** https://kyuz0.github.io/amd-strix-halo-toolboxes/")
|
||||
lines.append("")
|
||||
lines.append("## Table of Contents")
|
||||
lines.append("- [Benchmark methodology](#benchmark-methodology)")
|
||||
lines.append("- [Summary of current dataset (Flash Attention ON)](#summary-of-current-dataset-flash-attention-on)")
|
||||
lines.append(" - [Placement counts](#placement-counts)")
|
||||
lines.append(" - [Pairwise head-to-head wins](#pairwise-head-to-head-wins)")
|
||||
lines.append(" - [Average ranks](#average-ranks)")
|
||||
lines.append("- [Analyses by feature](#analyses-by-feature)")
|
||||
lines.append(" - [Impact of Flash Attention](#impact-of-flash-attention)")
|
||||
lines.append(" - [Impact of ROCWMMA](#impact-of-rocwmma)")
|
||||
lines.append(" - [Impact of hipBLASLt](#impact-of-hipblaslt)")
|
||||
lines.append(" - [Vulkan: AMDVLK vs RADV](#vulkan-amdvlk-vs-radv)")
|
||||
lines.append("- [Recommendations](#recommendations)")
|
||||
lines.append("- [Winner calculation](#winner-calculation)")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Benchmark methodology")
|
||||
lines.append("")
|
||||
lines.append("- **pp512** — prompt processing throughput (tokens/sec, prefill)")
|
||||
lines.append("- **tg128** — token generation throughput (tokens/sec, interactive)")
|
||||
lines.append("- Each backend tested twice per model: `-fa 0` and `-fa 1`")
|
||||
lines.append("- Winners per model/test are **margin-aware**; multiple winners are possible when mean±σ overlap")
|
||||
lines.append("- Built from the same llama.cpp commit for consistency")
|
||||
lines.append("")
|
||||
lines.append("**Backends in this dataset:** " + ", ".join(ENV_LABEL.get(e, e) for e in envs))
|
||||
lines.append("")
|
||||
lines.append("**ROCm 7 hipBLASLt policy:** Toolboxes ship with **hipBLASLt enabled** by default (`ROCBLAS_USE_HIPBLASLT=1`). The benchmark script also runs **hipBLASLt OFF** variants (`-hblt0`) to measure its effect.")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Summary of current dataset (Flash Attention ON)")
|
||||
lines.append("")
|
||||
# Placement counts
|
||||
lines.append("### Placement counts")
|
||||
def place_block(title: str, place_dict: Dict[str, Dict[str, int]]):
|
||||
lines.append(f"**{title}**")
|
||||
lines.append(md_row(["Backend", "1st", "2nd", "3rd"]))
|
||||
lines.append(md_row(["---", "---:", "---:", "---:"]))
|
||||
order = sorted(place_dict.items(), key=lambda kv: (-kv[1].get("first", 0), -kv[1].get("second", 0), kv[0]))
|
||||
for env, c in order:
|
||||
lines.append(md_row([ENV_LABEL.get(env, env), str(c.get("first", 0)), str(c.get("second", 0)), str(c.get("third", 0))]))
|
||||
lines.append("")
|
||||
place_block("Prompt Processing (pp512)", pp_place)
|
||||
place_block("Token Generation (tg128)", tg_place)
|
||||
|
||||
# Pairwise wins
|
||||
lines.append("### Pairwise head-to-head wins")
|
||||
lines.append("For any model+quant where both backends succeeded, this counts who was faster (ties when equal).")
|
||||
lines.append(md_row(["Comparison", "Test", "A wins", "B wins", "Ties", "Total"]))
|
||||
lines.append(md_row(["---", "---", "---:", "---:", "---:", "---:"]))
|
||||
pairs = [
|
||||
("ROCm 7 RC + ROCWMMA + hipBLASLt", "Vulkan AMDVLK", "rocm7_rc-rocwmma", "vulkan_amdvlk"),
|
||||
("ROCm 7 RC + ROCWMMA + hipBLASLt", "Vulkan RADV", "rocm7_rc-rocwmma", "vulkan_radv"),
|
||||
("Vulkan AMDVLK", "Vulkan RADV", "vulkan_amdvlk", "vulkan_radv"),
|
||||
]
|
||||
for labelA, labelB, envA, envB in pairs:
|
||||
for test in TESTS:
|
||||
a, b, t, total = pairwise_win_counts(runs, envA, envB, test, fa_filter)
|
||||
lines.append(md_row([f"{labelA} vs {labelB}", test, str(a), str(b), str(t), str(total)]))
|
||||
lines.append("")
|
||||
|
||||
# Average ranks
|
||||
lines.append("### Average ranks")
|
||||
avg_pp = average_ranks(pp_place)
|
||||
avg_tg = average_ranks(tg_place)
|
||||
lines.append("**Prompt Processing (pp512)**")
|
||||
lines.append(md_row(["Backend", "Avg Rank (↓ is better)"]))
|
||||
lines.append(md_row(["---", "---:"]))
|
||||
for env, val in sorted(avg_pp.items(), key=lambda kv: (kv[1] is None, kv[1] or 99)):
|
||||
lines.append(md_row([ENV_LABEL.get(env, env), str(val) if val is not None else "—"]))
|
||||
lines.append("")
|
||||
lines.append("**Token Generation (tg128)**")
|
||||
lines.append(md_row(["Backend", "Avg Rank (↓ is better)"]))
|
||||
lines.append(md_row(["---", "---:"]))
|
||||
for env, val in sorted(avg_tg.items(), key=lambda kv: (kv[1] is None, kv[1] or 99)):
|
||||
lines.append(md_row([ENV_LABEL.get(env, env), str(val) if val is not None else "—"]))
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Analyses by feature")
|
||||
lines.append("")
|
||||
|
||||
# Flash Attention effect
|
||||
lines.append("### Impact of Flash Attention")
|
||||
fa_eff = flash_attention_effect(runs, envs)
|
||||
lines.append("Median % change when **Flash Attention ON vs OFF**, paired by model+quant, per backend:")
|
||||
lines.append(md_row(["Backend", "pp512 Δ% (median, min..max, n)", "tg128 Δ% (median, min..max, n)"]))
|
||||
lines.append(md_row(["---", "---", "---"]))
|
||||
def fmt_eff(row: Optional[Dict[str, float]]) -> str:
|
||||
return f"{row['median_pct']}% ({row['min']}..{row['max']}), n={row['n_pairs']}" if row else "—"
|
||||
for env in envs:
|
||||
row_pp = fa_eff.get(env, {}).get("pp512")
|
||||
row_tg = fa_eff.get(env, {}).get("tg128")
|
||||
lines.append(md_row([ENV_LABEL.get(env, env), fmt_eff(row_pp), fmt_eff(row_tg)]))
|
||||
lines.append("")
|
||||
|
||||
# ROCWMMA effect — check both ROCm 7 and 6.4.4 families if present
|
||||
lines.append("### Impact of ROCWMMA")
|
||||
rocwmma_pairs = []
|
||||
if "rocm7_rc-rocwmma" in envs and "rocm7_rc" in envs:
|
||||
rocwmma_pairs.append(("rocm7_rc-rocwmma", "rocm7_rc", "ROCm 7 RC (hipBLASLt)"))
|
||||
if "rocm7_rc-rocwmma-hblt0" in envs and "rocm7_rc-hblt0" in envs:
|
||||
rocwmma_pairs.append(("rocm7_rc-rocwmma-hblt0", "rocm7_rc-hblt0", "ROCm 7 RC (hipBLASLt OFF)"))
|
||||
if "rocm6_4_4-rocwmma" in envs and "rocm6_4_4" in envs:
|
||||
rocwmma_pairs.append(("rocm6_4_4-rocwmma", "rocm6_4_4", "ROCm 6.4.4 (hipBLASLt)"))
|
||||
if "rocm6_4_4-rocwmma-hblt0" in envs and "rocm6_4_4-hblt0" in envs:
|
||||
rocwmma_pairs.append(("rocm6_4_4-rocwmma-hblt0", "rocm6_4_4-hblt0", "ROCm 6.4.4 (hipBLASLt OFF)"))
|
||||
|
||||
rocwmma_rows = rocwmma_effect(runs, rocwmma_pairs, TESTS)
|
||||
lines.append(md_row(["Context", "Test", "Compared Envs", "Pairs", "Median Δ%"]))
|
||||
lines.append(md_row(["---", "---", "---", "---:", "---:"]))
|
||||
for label, test, env_on, env_off, n, delta in rocwmma_rows:
|
||||
lines.append(md_row([label, test, f"{ENV_LABEL.get(env_on, env_on)} vs {ENV_LABEL.get(env_off, env_off)}", str(n), f"{delta}%"]))
|
||||
lines.append("")
|
||||
|
||||
# hipBLASLt effect — for both ROCm 7 and 6.4.4 families
|
||||
lines.append("### Impact of hipBLASLt")
|
||||
hip_pairs = []
|
||||
if "rocm7_rc" in envs and "rocm7_rc-hblt0" in envs:
|
||||
hip_pairs.append(("rocm7_rc", "rocm7_rc-hblt0", "ROCm 7 RC (no ROCWMMA)"))
|
||||
if "rocm7_rc-rocwmma" in envs and "rocm7_rc-rocwmma-hblt0" in envs:
|
||||
hip_pairs.append(("rocm7_rc-rocwmma", "rocm7_rc-rocwmma-hblt0", "ROCm 7 RC + ROCWMMA"))
|
||||
if "rocm6_4_4" in envs and "rocm6_4_4-hblt0" in envs:
|
||||
hip_pairs.append(("rocm6_4_4", "rocm6_4_4-hblt0", "ROCm 6.4.4 (no ROCWMMA)"))
|
||||
if "rocm6_4_4-rocwmma" in envs and "rocm6_4_4-rocwmma-hblt0" in envs:
|
||||
hip_pairs.append(("rocm6_4_4-rocwmma", "rocm6_4_4-rocwmma-hblt0", "ROCm 6.4.4 + ROCWMMA"))
|
||||
|
||||
hip_rows = hipblaslt_effect(runs, hip_pairs, TESTS)
|
||||
lines.append(md_row(["Context", "Test", "Compared Envs", "Pairs", "Median Δ%"]))
|
||||
lines.append(md_row(["---", "---", "---", "---:", "---:"]))
|
||||
for label, test, env_on, env_off, n, delta in hip_rows:
|
||||
lines.append(md_row([label, test, f"{ENV_LABEL.get(env_on, env_on)} vs {ENV_LABEL.get(env_off, env_off)}", str(n), f"{delta}%"]))
|
||||
lines.append("")
|
||||
|
||||
# AMDVLK vs RADV
|
||||
lines.append("### Vulkan: AMDVLK vs RADV")
|
||||
lines.append("Head-to-head wins with selected Flash Attention filter:")
|
||||
lines.append(md_row(["Test", "AMDVLK wins", "RADV wins", "Ties", "Total"]))
|
||||
lines.append(md_row(["---", "---:", "---:", "---:", "---:"]))
|
||||
for test, wa, wr, t, total in amdvlk_vs_radv(runs, fa_filter):
|
||||
lines.append(md_row([test, str(wa), str(wr), str(t), str(total)]))
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Recommendations")
|
||||
pp_wins, _ = winners(pp_place, "first")
|
||||
tg_wins, _ = winners(tg_place, "first")
|
||||
lines.append(f"- **Fastest prompt processing:** {human_list(pp_wins)} (most 1st-place finishes with selected Flash Attention filter).")
|
||||
lines.append(f"- **Fastest token generation:** {human_list(tg_wins)} (most 1st-place finishes with selected Flash Attention filter).")
|
||||
# Balanced: highest (2*first + second) across PP+TG
|
||||
def score(c: Dict[str, int]) -> int:
|
||||
return c.get("first", 0) * 2 + c.get("second", 0)
|
||||
best_bal = -1
|
||||
balanced: List[str] = []
|
||||
for env in envs:
|
||||
s = score(pp_place.get(env, {})) + score(tg_place.get(env, {}))
|
||||
if s > best_bal:
|
||||
best_bal = s
|
||||
balanced = [env]
|
||||
elif s == best_bal:
|
||||
balanced.append(env)
|
||||
lines.append(f"- **Balanced choice:** {human_list(balanced)} (consistently near the top across PP/TG).")
|
||||
lines.append("")
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
lines.append("## Winner calculation")
|
||||
lines.append("A backend is counted as a winner if its mean throughput is within the best backend’s pooled ± error margin for that model/test type. This treats results within measurement noise as ties instead of false losses.")
|
||||
return "\n".join(lines)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--file", type=Path, default=Path("../docs/results.json"),
|
||||
help="Path to results.json (default: ../docs/results.json)")
|
||||
ap.add_argument("--out-readme", type=Path, default=Path("./README_benchmarks_section.md"),
|
||||
help="Path to write README section Markdown (default: ./README_benchmarks_section.md)")
|
||||
ap.add_argument("--out-bench", type=Path, default=Path("./benchmarks_generated.md"),
|
||||
help="Path to write detailed benchmarks Markdown (default: ./benchmarks_generated.md)")
|
||||
ap.add_argument("--fa", choices=["on", "off", "any"], default="on",
|
||||
help="Flash Attention filter (default: on)")
|
||||
ap.add_argument("--include-all-envs", action="store_true",
|
||||
help="Include envs even if not present in results.json")
|
||||
ap.add_argument("--only-env", action="append",
|
||||
help="Restrict analysis to specific env keys (repeatable)")
|
||||
args = ap.parse_args()
|
||||
|
||||
data = load_results(args.file)
|
||||
runs: List[Dict] = data["runs"]
|
||||
fa_filter = fa_to_filter(args.fa)
|
||||
envs = envs_present(runs, args.only_env, args.include_all_envs)
|
||||
|
||||
pp_place, _ = margin_aware_placements(runs, envs, "pp512", fa_filter)
|
||||
tg_place, _ = margin_aware_placements(runs, envs, "tg128", fa_filter)
|
||||
|
||||
readme_md = build_readme_section(envs, pp_place, tg_place, fa_filter)
|
||||
args.out_readme.write_text(readme_md)
|
||||
|
||||
bench_md = build_benchmarks_doc(runs, envs, pp_place, tg_place, fa_filter)
|
||||
args.out_bench.write_text(bench_md)
|
||||
|
||||
print(f"Wrote:\n - {args.out_readme}\n - {args.out_bench}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
import re, glob, os, json, time
|
||||
from pathlib import Path
|
||||
|
||||
RESULT_SOURCES = [
|
||||
("results", False), # regular single-node runs
|
||||
("results-rpc", True), # distributed RPC runs across two servers
|
||||
]
|
||||
OUT_JSON = "../docs/results.json"
|
||||
|
||||
# --- Regexes ---------------------------------------------------------------
|
||||
|
||||
# Table headers come in two shapes (with or without "fa" column)
|
||||
HEADER_RE = re.compile(r"^\|\s*model\s*\|", re.IGNORECASE)
|
||||
SEP_RE = re.compile(r"^\|\s*-+")
|
||||
|
||||
# Build line, e.g. "build: cd6983d5 (6119)"
|
||||
BUILD_RE = re.compile(r"build:\s*([0-9a-f]{7,})\s*\((\d+)\)", re.IGNORECASE)
|
||||
|
||||
# Error classifiers (same spirit as your table script)
|
||||
LOAD_ERR = re.compile(r"failed to load model|Device memory allocation.*failed|⚠️\s*Fail", re.IGNORECASE)
|
||||
HANG_ERR = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
|
||||
GENERIC_ERR= re.compile(r"error:|exit \d+|runtime error|⚠️\s*Runtime Error", re.IGNORECASE)
|
||||
|
||||
# Extract numeric ± numeric from the last column
|
||||
TS_RE = re.compile(r"([\d.]+)\s*±\s*([\d.]+)")
|
||||
|
||||
# Quantization from model name
|
||||
QUANT_RE = re.compile(r"(Q\d+_[A-Z0-9_]+|BF16|F16|F32|mxfp\d+)", re.IGNORECASE)
|
||||
|
||||
PARAMS_RE = re.compile(r"([\d.,]+)\s*B", re.IGNORECASE)
|
||||
GIB_RE = re.compile(r"([\d.,]+)\s*GiB", re.IGNORECASE)
|
||||
|
||||
# "30B", "235B" from model name
|
||||
NAME_B_RE = re.compile(r"(\d+(?:\.\d+)?)B")
|
||||
|
||||
# Shard suffix in filenames
|
||||
SHARD_RE = re.compile(r"-000\d+-of-000\d+", re.IGNORECASE)
|
||||
|
||||
# Long-context suffix in filenames (e.g., __longctx32768)
|
||||
LONGCTX_RE = re.compile(r"longctx(\d+)", re.IGNORECASE)
|
||||
|
||||
# --- Helpers ---------------------------------------------------------------
|
||||
|
||||
ENV_CANON = {
|
||||
"rocm7_1": "rocm7.1",
|
||||
"rocm7_alpha": "rocm-7alpha",
|
||||
}
|
||||
|
||||
def clean_model_name(raw):
|
||||
base = SHARD_RE.sub("", raw)
|
||||
return base
|
||||
|
||||
def canonicalize_env(env):
|
||||
if not env:
|
||||
return env
|
||||
for raw, canon in ENV_CANON.items():
|
||||
prefix = f"{raw}-"
|
||||
if env == raw:
|
||||
return canon
|
||||
if env.startswith(prefix):
|
||||
return canon + env[len(raw):]
|
||||
return env
|
||||
|
||||
def parse_env_flags(basename):
|
||||
"""
|
||||
pattern: <model>__<env>[__fa1][__hblt0][__longctx32768][__rpc]
|
||||
Returns (env, fa, context_tag, context_tokens, rpc_flag)
|
||||
"""
|
||||
parts = basename.split("__")
|
||||
if len(parts) < 2:
|
||||
return None, False, "default", None, False
|
||||
|
||||
env = parts[1]
|
||||
fa = False
|
||||
context_tag = "default"
|
||||
context_tokens = None
|
||||
rpc_flag = False
|
||||
|
||||
for raw_suffix in parts[2:]:
|
||||
suffix = raw_suffix.lower()
|
||||
if suffix == "fa1":
|
||||
fa = True
|
||||
elif suffix == "hblt0":
|
||||
env = f"{env}-hblt0"
|
||||
elif suffix.startswith("longctx"):
|
||||
context_tag = suffix
|
||||
m = LONGCTX_RE.search(suffix)
|
||||
if m:
|
||||
try:
|
||||
context_tokens = int(m.group(1))
|
||||
except ValueError:
|
||||
context_tokens = None
|
||||
elif suffix == "rpc":
|
||||
rpc_flag = True
|
||||
|
||||
return env, fa, context_tag, context_tokens, rpc_flag
|
||||
|
||||
def env_base_and_variant(env):
|
||||
# e.g. "rocm6_4_2-rocwmma" -> ("rocm6_4_2", "rocwmma")
|
||||
if "-" in env:
|
||||
base, variant = env.split("-", 1)
|
||||
return base, variant
|
||||
return env, None
|
||||
|
||||
def detect_error(text):
|
||||
if LOAD_ERR.search(text):
|
||||
return True, "load"
|
||||
if HANG_ERR.search(text):
|
||||
return True, "hang"
|
||||
if GENERIC_ERR.search(text):
|
||||
return True, "runtime"
|
||||
return False, None
|
||||
|
||||
def parse_table(text):
|
||||
"""
|
||||
Returns list of rows parsed from the markdown-like table.
|
||||
Each row is a dict of the parsed columns, normalized by header names.
|
||||
Handles presence/absence of the 'fa' column.
|
||||
"""
|
||||
lines = text.splitlines()
|
||||
rows = []
|
||||
header = None
|
||||
col_idx = {}
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if HEADER_RE.search(line):
|
||||
# header line
|
||||
header = [c.strip().lower() for c in line.strip().strip("|").split("|")]
|
||||
# next line should be the separator; skip it
|
||||
# build index map
|
||||
for idx, name in enumerate(header):
|
||||
col_idx[name] = idx
|
||||
continue
|
||||
if header and (SEP_RE.search(line) or not line.strip()):
|
||||
# skip separators / blanks after header
|
||||
continue
|
||||
if header and line.startswith("|"):
|
||||
parts = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
# guard for short lines
|
||||
if len(parts) < len(header):
|
||||
continue
|
||||
row = {}
|
||||
for name, idx in col_idx.items():
|
||||
row[name] = parts[idx]
|
||||
rows.append(row)
|
||||
# stop parsing block when a blank line after some rows appears
|
||||
if header and line.strip() == "" and rows:
|
||||
break
|
||||
|
||||
return rows
|
||||
|
||||
def coerce_float(m, default=None):
|
||||
try:
|
||||
return float(m)
|
||||
except:
|
||||
return default
|
||||
|
||||
def extract_quant(model_name):
|
||||
m = QUANT_RE.search(model_name)
|
||||
return (m.group(1).upper() if m else None)
|
||||
|
||||
def b_from_name(model_name):
|
||||
m = NAME_B_RE.search(model_name)
|
||||
return coerce_float(m.group(1)) if m else None
|
||||
|
||||
# --- Main scan -------------------------------------------------------------
|
||||
|
||||
runs = []
|
||||
builds = set()
|
||||
envs = set()
|
||||
|
||||
for results_dir, is_rpc_source in RESULT_SOURCES:
|
||||
glob_pattern = os.path.join(results_dir, "*.log")
|
||||
for path in sorted(glob.glob(glob_pattern)):
|
||||
base = os.path.basename(path).rsplit(".log", 1)[0]
|
||||
if "__" not in base:
|
||||
continue
|
||||
|
||||
model_raw, _rest = base.split("__", 1)
|
||||
env, fa_from_name, context_tag, context_tokens, rpc_flag = parse_env_flags(base)
|
||||
env = canonicalize_env(env)
|
||||
if env:
|
||||
envs.add(env)
|
||||
|
||||
model_clean = clean_model_name(model_raw)
|
||||
|
||||
with open(path, errors="ignore") as f:
|
||||
text = f.read()
|
||||
|
||||
# build info (take the last match in file if many)
|
||||
build_hash, build_num = None, None
|
||||
for m in BUILD_RE.finditer(text):
|
||||
build_hash, build_num = m.group(1), m.group(2)
|
||||
if build_hash:
|
||||
builds.add((build_hash, build_num))
|
||||
|
||||
# detect error (if there is no valid table rows)
|
||||
table_rows = parse_table(text)
|
||||
|
||||
# If table rows exist, we’ll still mark errors only if no perf found
|
||||
has_pp = any(r.get("test","").lower()=="pp512" for r in table_rows)
|
||||
has_tg = any(r.get("test","").lower()=="tg128" for r in table_rows)
|
||||
error, etype = (False, None)
|
||||
if not (has_pp or has_tg):
|
||||
error, etype = detect_error(text)
|
||||
|
||||
# Determine FA flag:
|
||||
# prefer explicit column "fa" if present, else fallback to filename "__fa1"
|
||||
fa_in_table = None
|
||||
for r in table_rows:
|
||||
if "fa" in r:
|
||||
try:
|
||||
fa_in_table = int(r["fa"]) == 1
|
||||
except:
|
||||
fa_in_table = None
|
||||
break
|
||||
fa_enabled = fa_in_table if fa_in_table is not None else fa_from_name
|
||||
|
||||
# Normalize env base / variant (e.g., rocwmma)
|
||||
env_base, env_variant = env_base_and_variant(env)
|
||||
|
||||
# Emit one run per row (pp512 / tg128)
|
||||
for r in table_rows or [{}]:
|
||||
test = r.get("test", "").lower() if table_rows else None
|
||||
tps_mean, tps_std = None, None
|
||||
if table_rows:
|
||||
ts_field = r.get("t/s", "")
|
||||
m = TS_RE.search(ts_field)
|
||||
if m:
|
||||
tps_mean = coerce_float(m.group(1))
|
||||
tps_std = coerce_float(m.group(2))
|
||||
|
||||
# parse numeric helpers from row (if present)
|
||||
params_b = None
|
||||
file_size_gib = None
|
||||
if "params" in r:
|
||||
pm = PARAMS_RE.search(r["params"])
|
||||
if pm:
|
||||
params_b = coerce_float(pm.group(1).replace(",", ""))
|
||||
if "size" in r:
|
||||
sm = GIB_RE.search(r["size"])
|
||||
if sm:
|
||||
file_size_gib = coerce_float(sm.group(1).replace(",", ""))
|
||||
|
||||
# quant from model name (unchanged)
|
||||
quant = extract_quant(model_clean)
|
||||
|
||||
# name_params_b: prefer table value; else fall back to B in model name
|
||||
name_params_b = params_b if params_b is not None else b_from_name(model_clean)
|
||||
|
||||
backend = r.get("backend")
|
||||
ngl = r.get("ngl")
|
||||
mmap = r.get("mmap")
|
||||
|
||||
run = {
|
||||
"model": model_raw,
|
||||
"model_clean": model_clean,
|
||||
"env": env,
|
||||
"env_base": env_base,
|
||||
"env_variant": env_variant, # e.g. "rocwmma"
|
||||
"fa": bool(fa_enabled),
|
||||
"context": context_tag or "default",
|
||||
"context_tokens": context_tokens,
|
||||
"test": test, # "pp512" | "tg128" | None (if error)
|
||||
"tps_mean": tps_mean,
|
||||
"tps_std": tps_std,
|
||||
"error": bool(error),
|
||||
"error_type": etype, # "load" | "hang" | "runtime" | None
|
||||
"backend": backend,
|
||||
"ngl": (int(ngl) if (ngl and ngl.isdigit()) else None),
|
||||
"mmap": (int(mmap) if (mmap and mmap.isdigit()) else None),
|
||||
"params_b": params_b, # from table, if available
|
||||
"file_size_gib": file_size_gib, # from table, if available
|
||||
"name_params_b": name_params_b, # parsed from model name (e.g., 30B -> 30.0)
|
||||
"quant": quant,
|
||||
"log": path,
|
||||
"rpc": bool(is_rpc_source or rpc_flag),
|
||||
"build": {"hash": build_hash, "number": build_num} if build_hash else None,
|
||||
}
|
||||
runs.append(run)
|
||||
|
||||
# Meta
|
||||
meta = {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"os_kernel": "Fedora 42 — Linux 6.15.9-201.fc42.x86_64 (Sat Aug 2 11:37:34 UTC 2025)",
|
||||
"llamacpp_builds": [{"hash": h, "number": n} for (h, n) in sorted(builds)],
|
||||
"environments": sorted(envs),
|
||||
"notes": "pp512 = prompt processing; tg128 = text generation; t/s = tokens/second",
|
||||
}
|
||||
|
||||
out = {"meta": meta, "runs": runs}
|
||||
|
||||
Path(OUT_JSON).write_text(json.dumps(out, indent=2))
|
||||
print(f"Wrote {OUT_JSON} with {len(runs)} rows.")
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
import re, glob, os
|
||||
|
||||
# This script parses llama-bench logs in 'results/' to produce
|
||||
# Markdown tables for pp512 (prompt processing) and tg128 (text generation).
|
||||
|
||||
# Regex patterns to extract tokens/sec rows
|
||||
PP_RE = re.compile(r"\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|\s*pp512\s*\|\s*([\d.]+)\s*±\s*([\d.]+)")
|
||||
TG_RE = re.compile(r"\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|\s*tg128\s*\|\s*([\d.]+)\s*±\s*([\d.]+)")
|
||||
|
||||
# Patterns to classify errors
|
||||
LOAD_ERR = re.compile(r"failed to load model|Device memory allocation.*failed", re.IGNORECASE)
|
||||
HANG_ERR = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
|
||||
GENERIC_ERR = re.compile(r"error:|exit \d+", re.IGNORECASE)
|
||||
|
||||
# Env ordering
|
||||
ENV_ORDER = ["vulkan_radv","vulkan_amdvlk","rocm6_4_2","rocm7_beta","rocm7_rc"]
|
||||
|
||||
data = {}
|
||||
|
||||
# Utility to clean model names
|
||||
def clean_name(raw):
|
||||
return re.sub(r"-000\d+-of-000\d+", "", raw)
|
||||
|
||||
# Scan logs
|
||||
glob_pattern = os.path.join("results", "*.log")
|
||||
for path in sorted(glob.glob(glob_pattern)):
|
||||
# Fix: use rsplit, not rssplit
|
||||
base = os.path.basename(path).rsplit('.log',1)[0]
|
||||
if '__' not in base:
|
||||
continue
|
||||
model_raw, env = base.split('__',1)
|
||||
model = clean_name(model_raw)
|
||||
|
||||
text = open(path, errors='ignore').read()
|
||||
# Determine error type
|
||||
if LOAD_ERR.search(text):
|
||||
err_type = 'load'
|
||||
elif HANG_ERR.search(text):
|
||||
err_type = 'hang'
|
||||
elif GENERIC_ERR.search(text) and not (PP_RE.search(text) and TG_RE.search(text)):
|
||||
err_type = 'runtime'
|
||||
else:
|
||||
err_type = None
|
||||
|
||||
# Extract performance if no load error
|
||||
pp_match = PP_RE.search(text) if err_type is None else None
|
||||
tg_match = TG_RE.search(text) if err_type is None else None
|
||||
|
||||
for key, match in [('pp512', pp_match), ('tg128', tg_match)]:
|
||||
cell = {
|
||||
'mean': match.group(1) if match else None,
|
||||
'std': match.group(2) if match else None,
|
||||
'error': err_type is not None,
|
||||
'etype': err_type
|
||||
}
|
||||
data.setdefault(model, {}).setdefault(key, {})[env] = cell
|
||||
|
||||
# Select winner
|
||||
def pick_winner(env_data):
|
||||
scores = {e: float(d['mean']) for e,d in env_data.items() if not d['error'] and d['mean']}
|
||||
if not scores:
|
||||
return '—'
|
||||
best = max(scores, key=scores.get)
|
||||
others = [v for k,v in scores.items() if k!=best]
|
||||
tag = f"🏆 **{best}**"
|
||||
if others:
|
||||
gain = (scores[best]/max(others)-1)*100
|
||||
tag += f" (+{gain:.0f}%)"
|
||||
return tag
|
||||
|
||||
# Render table with distinct error messages
|
||||
def render_table(test_label, display_name):
|
||||
print(f"### {display_name} — tokens/second\n")
|
||||
header = ['Model'] + [e.replace('_',' ').title() for e in ENV_ORDER] + ['Winner']
|
||||
print("| " + " | ".join(header) + " |")
|
||||
print("|" + "|".join(['---']*len(header)) + "|")
|
||||
|
||||
for model in sorted(data, key=lambda s: s.lower()):
|
||||
row = [f"**{model}**"]
|
||||
env_data = data[model].get(test_label, {})
|
||||
for env in ENV_ORDER:
|
||||
d = env_data.get(env)
|
||||
if not d:
|
||||
cell = '—'
|
||||
elif d['error']:
|
||||
et = d['etype']
|
||||
if et=='load':
|
||||
cell = '⚠️ Load Error'
|
||||
elif et=='hang':
|
||||
cell = '⚠️ GPU Hang'
|
||||
else:
|
||||
cell = '⚠️ Runtime Error'
|
||||
else:
|
||||
cell = f"{float(d['mean']):.2f} ± {float(d['std']):.2f}"
|
||||
row.append(cell)
|
||||
row.append(pick_winner(env_data))
|
||||
print("| " + " | ".join(row) + " |")
|
||||
print()
|
||||
|
||||
# Output tables
|
||||
render_table('pp512','Prompt Processing (pp512)')
|
||||
render_table('tg128','Text Generation (tg128)')
|
||||
|
||||
# Summary of failures by type
|
||||
fail_lines = []
|
||||
for model in sorted(data, key=lambda s: s.lower()):
|
||||
for test_label, envs in data[model].items():
|
||||
for env,d in envs.items():
|
||||
if d['error']:
|
||||
et = d['etype'] or 'unknown'
|
||||
desc = {
|
||||
'load':'failed to load',
|
||||
'hang':'GPU hang',
|
||||
'runtime':'runtime error',
|
||||
}.get(et, 'error')
|
||||
fail_lines.append(f"- **{model}** [{test_label}] on *{env}*: {desc}")
|
||||
if fail_lines:
|
||||
print("## Failed Runs\n")
|
||||
print("\n".join(fail_lines))
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
MODEL_DIR="$(realpath models)"
|
||||
RESULTDIR="results"
|
||||
mkdir -p "$RESULTDIR"
|
||||
|
||||
# Pick exactly one .gguf per model: either
|
||||
# - any .gguf without "-000*-of-" (single-file models)
|
||||
# - or the first shard "*-00001-of-*.gguf"
|
||||
mapfile -t MODEL_PATHS < <(
|
||||
find "$MODEL_DIR" -type f -name '*.gguf' \
|
||||
\( -name '*-00001-of-*.gguf' -o -not -name '*-000*-of-*.gguf' \) \
|
||||
| sort
|
||||
)
|
||||
|
||||
if (( ${#MODEL_PATHS[@]} == 0 )); then
|
||||
echo "❌ No models found under $MODEL_DIR – check your paths/patterns!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found ${#MODEL_PATHS[@]} model(s) to bench:"
|
||||
for p in "${MODEL_PATHS[@]}"; do
|
||||
echo " • $p"
|
||||
done
|
||||
echo
|
||||
|
||||
declare -A CMDS=(
|
||||
[rocm6_4_4]="toolbox run -c llama-rocm-6.4.4 -- /usr/local/bin/llama-bench"
|
||||
[rocm6_4_4-rocwmma]="toolbox run -c llama-rocm-6.4.4-rocwmma -- /usr/local/bin/llama-bench"
|
||||
[rocm7.1]="toolbox run -c llama-rocm-7.1 -- /usr/local/bin/llama-bench"
|
||||
[rocm7.1-rocwmma]="toolbox run -c llama-rocm-7.1-rocwmma -- /usr/local/bin/llama-bench"
|
||||
[rocm-7alpha-rocwmma-improved]="toolbox run -c llama-rocm-7alpha-rocwmma-improved -- /usr/local/bin/llama-bench"
|
||||
[rocm-7alpha]="toolbox run -c llama-rocm-7alpha -- /usr/local/bin/llama-bench"
|
||||
[rocm-7alpha-rocwmma]="toolbox run -c llama-rocm-7alpha-rocwmma -- /usr/local/bin/llama-bench"
|
||||
[rocm7_rc]="toolbox run -c llama-rocm-7rc -- /usr/local/bin/llama-bench"
|
||||
[rocm7_rc-rocwmma]="toolbox run -c llama-rocm-7rc-rocwmma -- /usr/local/bin/llama-bench"
|
||||
[vulkan_amdvlk]="toolbox run -c llama-vulkan-amdvlk -- /usr/sbin/llama-bench"
|
||||
[vulkan_radv]="toolbox run -c llama-vulkan-radv -- /usr/sbin/llama-bench"
|
||||
)
|
||||
|
||||
get_hblt_modes() {
|
||||
local env="$1"
|
||||
if [[ "$env" == rocm* ]]; then
|
||||
printf '%s\n' default off
|
||||
else
|
||||
printf '%s\n' default
|
||||
fi
|
||||
}
|
||||
|
||||
for MODEL_PATH in "${MODEL_PATHS[@]}"; do
|
||||
MODEL_NAME="$(basename "$MODEL_PATH" .gguf)"
|
||||
|
||||
for ENV in "${!CMDS[@]}"; do
|
||||
CMD="${CMDS[$ENV]}"
|
||||
mapfile -t HBLT_MODES < <(get_hblt_modes "$ENV")
|
||||
|
||||
for MODE in "${HBLT_MODES[@]}"; do
|
||||
BASE_SUFFIX=""
|
||||
CMD_EFFECTIVE="$CMD"
|
||||
|
||||
if [[ "$ENV" == rocm* ]]; then
|
||||
if [[ "$MODE" == off ]]; then
|
||||
BASE_SUFFIX="__hblt0"
|
||||
CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=0 }"
|
||||
else
|
||||
CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=1 }"
|
||||
fi
|
||||
fi
|
||||
|
||||
# run twice: baseline and with flash attention
|
||||
for FA in 1; do
|
||||
SUFFIX="$BASE_SUFFIX"
|
||||
EXTRA_ARGS=()
|
||||
if (( FA == 1 )); then
|
||||
SUFFIX="${SUFFIX}__fa1"
|
||||
EXTRA_ARGS=( -fa 1 )
|
||||
fi
|
||||
|
||||
for CTX in default longctx32768; do
|
||||
CTX_SUFFIX=""
|
||||
CTX_ARGS=()
|
||||
if [[ "$CTX" == longctx32768 ]]; then
|
||||
CTX_SUFFIX="__longctx32768"
|
||||
CTX_ARGS=( -p 2048 -n 32 -d 32768 )
|
||||
if [[ "$ENV" == *vulkan* ]]; then
|
||||
CTX_ARGS+=( -ub 512 )
|
||||
else
|
||||
CTX_ARGS+=( -ub 2048 )
|
||||
fi
|
||||
fi
|
||||
|
||||
OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}.log"
|
||||
CTX_REPS=3
|
||||
if [[ "$CTX" == longctx32768 ]]; then
|
||||
CTX_REPS=1
|
||||
fi
|
||||
|
||||
if [[ -s "$OUT" ]]; then
|
||||
echo "⏩ Skipping [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ ($CTX_SUFFIX)}, log already exists at $OUT"
|
||||
continue
|
||||
fi
|
||||
|
||||
FULL_CMD=( $CMD_EFFECTIVE -ngl 99 -mmp 0 -m "$MODEL_PATH" "${EXTRA_ARGS[@]}" "${CTX_ARGS[@]}" -r "$CTX_REPS" )
|
||||
|
||||
printf "\n▶ [%s] %s%s%s\n" "$ENV" "$MODEL_NAME" "${SUFFIX:+ $SUFFIX}" "${CTX_SUFFIX:+ $CTX_SUFFIX}"
|
||||
printf " → log: %s\n" "$OUT"
|
||||
printf " → cmd: %s\n\n" "${FULL_CMD[*]}"
|
||||
|
||||
if ! "${FULL_CMD[@]}" >"$OUT" 2>&1; then
|
||||
status=$?
|
||||
echo "✖ ! [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ $CTX_SUFFIX} failed (exit ${status})" >>"$OUT"
|
||||
echo " * [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ $CTX_SUFFIX} : FAILED"
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,666 @@
|
||||
:root {
|
||||
--bg: #f5f6fa;
|
||||
--ink: #101828;
|
||||
--muted: #6b7080;
|
||||
--accent: #155eef;
|
||||
--border: #d8dce6;
|
||||
--card: #ffffff;
|
||||
--chip-bg: #e6ecff;
|
||||
--chip-active-bg: #155eef;
|
||||
--chip-active-ink: #fff;
|
||||
--winner-bg: #d7f5e3;
|
||||
--winner-ink: #025333;
|
||||
--warn: #c2410c;
|
||||
--model-col: 180px;
|
||||
--winner-col: 120px;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
font: 13px/1.35 "Inter", "Segoe UI", system-ui, -apple-system, sans-serif;
|
||||
background: var(--bg);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
header {
|
||||
padding: 14px 20px 4px;
|
||||
background: var(--card);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
header h1 {
|
||||
margin: 0 0 4px;
|
||||
font-size: 20px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
header p {
|
||||
margin: 2px 0;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.controls,
|
||||
.panel {
|
||||
background: var(--card);
|
||||
border-bottom: 1px solid var(--border);
|
||||
padding: 10px 20px;
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
flex-wrap: wrap;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.control {
|
||||
min-width: 200px;
|
||||
}
|
||||
|
||||
.control.grow {
|
||||
flex: 1 1 320px;
|
||||
}
|
||||
|
||||
.slider-block {
|
||||
min-width: 260px;
|
||||
}
|
||||
|
||||
label {
|
||||
display: block;
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--muted);
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
|
||||
input[type="text"],
|
||||
select {
|
||||
width: 100%;
|
||||
padding: 6px 9px;
|
||||
border-radius: 6px;
|
||||
border: 1px solid var(--border);
|
||||
font-size: 13px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.chip-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.chip {
|
||||
border: none;
|
||||
border-radius: 999px;
|
||||
padding: 3px 10px;
|
||||
font-size: 12px;
|
||||
cursor: pointer;
|
||||
background: var(--chip-bg);
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.chip.active {
|
||||
background: var(--chip-active-bg);
|
||||
color: var(--chip-active-ink);
|
||||
}
|
||||
|
||||
.chip.small {
|
||||
font-size: 11px;
|
||||
padding: 3px 8px;
|
||||
}
|
||||
|
||||
.panel.compact {
|
||||
padding: 8px 20px;
|
||||
}
|
||||
|
||||
.panel-split {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
flex-wrap: wrap;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.backend-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px 14px;
|
||||
}
|
||||
|
||||
.backend-label {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.backend-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.backend-item {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
color: var(--ink);
|
||||
text-transform: none;
|
||||
}
|
||||
|
||||
.backend-item input {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.backend-item .tag {
|
||||
font-size: 10px;
|
||||
padding: 0 6px;
|
||||
border-radius: 999px;
|
||||
background: #eef2ff;
|
||||
color: #1d3ea5;
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
|
||||
.backend-item .tag.tag-hblt0 {
|
||||
background: #e9edff;
|
||||
color: #1d3ea5;
|
||||
}
|
||||
|
||||
.backend-item .tag.tag-rocwmma {
|
||||
background: #eef9ff;
|
||||
color: #0a517a;
|
||||
}
|
||||
|
||||
.backend-item .tag.tag-rocwmma-improved {
|
||||
background: #faf3ff;
|
||||
color: #6b1fb7;
|
||||
}
|
||||
|
||||
.backend-item .tag.tag-improved {
|
||||
background: #fef9e7;
|
||||
color: #8a5a00;
|
||||
}
|
||||
|
||||
.stats-box {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
#tables {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.test-block h2 {
|
||||
margin: 0 0 4px;
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
|
||||
.table-wrap {
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--card);
|
||||
position: relative;
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.table-scroll {
|
||||
overflow-x: auto;
|
||||
overflow-y: hidden;
|
||||
width: 100%;
|
||||
position: relative;
|
||||
scrollbar-gutter: stable both-edges;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.table-scroll table {
|
||||
min-width: 100%;
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
font-size: 11.5px;
|
||||
width: max-content;
|
||||
min-width: 100%;
|
||||
table-layout: fixed;
|
||||
}
|
||||
|
||||
thead {
|
||||
background: #f4f6fb;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: 4px 6px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
white-space: normal;
|
||||
border-right: 1px solid var(--border);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
th {
|
||||
position: relative;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
th.sticky,
|
||||
td.sticky {
|
||||
position: sticky;
|
||||
left: 0;
|
||||
background: inherit;
|
||||
z-index: 3;
|
||||
box-shadow: 1px 0 0 var(--border);
|
||||
}
|
||||
|
||||
th.model,
|
||||
td.model {
|
||||
width: var(--model-col);
|
||||
position: sticky;
|
||||
left: 0;
|
||||
z-index: 3;
|
||||
background: #f8f9ff;
|
||||
}
|
||||
|
||||
th.winner,
|
||||
td.winner {
|
||||
width: var(--winner-col);
|
||||
position: sticky;
|
||||
left: var(--model-col);
|
||||
z-index: 3;
|
||||
background: #f1f5ff;
|
||||
}
|
||||
|
||||
td.model {
|
||||
min-width: 170px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
td.model .model-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.model-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 2px 8px;
|
||||
border-radius: 999px;
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
background: #eceff5;
|
||||
color: #27303f;
|
||||
border: 1px solid transparent;
|
||||
}
|
||||
|
||||
.model-pill-rpc {
|
||||
background: #fdf2f8;
|
||||
border-color: #fbcfe8;
|
||||
color: #9d174d;
|
||||
}
|
||||
|
||||
.model-pill-rocwmma {
|
||||
background: #eef9ff;
|
||||
border-color: #c7e9ff;
|
||||
color: #0a517a;
|
||||
}
|
||||
|
||||
.legend {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.legend label {
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.legend-pills {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.legend-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
border-radius: 999px;
|
||||
border: 1px solid transparent;
|
||||
background: #e9edff;
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.legend-pill-default {
|
||||
background: #e9edff;
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.legend-pill-rpc {
|
||||
background: #fdf2f8;
|
||||
border-color: #fbcfe8;
|
||||
color: #9d174d;
|
||||
}
|
||||
|
||||
.legend-pill-rocwmma {
|
||||
background: #eef9ff;
|
||||
border-color: #c7e9ff;
|
||||
color: #0a517a;
|
||||
}
|
||||
|
||||
.legend-pill-rocwmma-improved {
|
||||
background: #faf3ff;
|
||||
border-color: #e0c8ff;
|
||||
color: #6b1fb7;
|
||||
}
|
||||
|
||||
.modal.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.modal {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 20px;
|
||||
z-index: 1000;
|
||||
}
|
||||
|
||||
.modal-content {
|
||||
background: #fff;
|
||||
border-radius: 12px;
|
||||
padding: 20px 24px;
|
||||
max-width: 520px;
|
||||
width: 100%;
|
||||
box-shadow: 0 12px 50px rgba(0, 0, 0, 0.2);
|
||||
position: relative;
|
||||
font-size: 13px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.modal-content h2 {
|
||||
margin-top: 0;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.modal-content p {
|
||||
margin: 8px 0;
|
||||
}
|
||||
|
||||
.modal-close {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 10px;
|
||||
border: none;
|
||||
background: transparent;
|
||||
font-size: 20px;
|
||||
cursor: pointer;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.modal-close:hover {
|
||||
color: var(--ink);
|
||||
}
|
||||
|
||||
.data-cell {
|
||||
white-space: normal;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.data-cell[data-env]:hover::after {
|
||||
content: attr(data-env);
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
transform: translateY(-50%);
|
||||
left: 50%;
|
||||
transform: translate(-50%, -120%);
|
||||
background: rgba(16, 24, 40, 0.92);
|
||||
color: #fff;
|
||||
padding: 4px 8px;
|
||||
border-radius: 6px;
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
pointer-events: none;
|
||||
z-index: 5;
|
||||
}
|
||||
|
||||
.data-cell[data-env]:hover::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -30%);
|
||||
border: 6px solid transparent;
|
||||
border-top-color: rgba(16, 24, 40, 0.92);
|
||||
pointer-events: none;
|
||||
z-index: 5;
|
||||
}
|
||||
|
||||
.data-cell .measure,
|
||||
.data-cell .std {
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.row-actions {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.row-action-btn {
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--accent);
|
||||
font-size: 11px;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
text-decoration: underline;
|
||||
text-underline-offset: 2px;
|
||||
}
|
||||
|
||||
.row-action-btn:hover {
|
||||
color: #0d3fb8;
|
||||
}
|
||||
|
||||
td.model .meta {
|
||||
font-size: 10px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
tbody tr:nth-child(even) td {
|
||||
background: #fafbff;
|
||||
}
|
||||
|
||||
.measure {
|
||||
font-feature-settings: "tnum";
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.std {
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.winner-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 2px;
|
||||
}
|
||||
|
||||
.winner-pill {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 2px 6px;
|
||||
border-radius: 999px;
|
||||
font-size: 10px;
|
||||
background: #dbeafe;
|
||||
color: #1e3a8a;
|
||||
margin: 1px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.cell-error {
|
||||
color: var(--warn);
|
||||
}
|
||||
|
||||
.cell-empty {
|
||||
color: #c3c7d1;
|
||||
}
|
||||
|
||||
.best {
|
||||
background: var(--winner-bg) !important;
|
||||
color: var(--winner-ink);
|
||||
}
|
||||
|
||||
td.best .measure,
|
||||
td.best .std {
|
||||
color: var(--winner-ink);
|
||||
}
|
||||
|
||||
.resize-handle {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
width: 6px;
|
||||
height: 100%;
|
||||
cursor: col-resize;
|
||||
}
|
||||
|
||||
.resize-handle::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
th.backend-header {
|
||||
cursor: grab;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
th.backend-header.dragging {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
th.backend-header.drop-target {
|
||||
outline: 2px dashed var(--accent);
|
||||
}
|
||||
|
||||
.resize-line {
|
||||
width: 2px;
|
||||
background: var(--accent);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.resize-overlay {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.resize-bar {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 6px;
|
||||
cursor: col-resize;
|
||||
pointer-events: auto;
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0 6px;
|
||||
border-radius: 999px;
|
||||
background: #f1f5ff;
|
||||
color: #1d4ed8;
|
||||
font-size: 11px;
|
||||
}
|
||||
.range-wrap {
|
||||
position: relative;
|
||||
height: 32px;
|
||||
}
|
||||
|
||||
.range-wrap input[type="range"] {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
background: transparent;
|
||||
-webkit-appearance: none;
|
||||
appearance: none;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.range-wrap input[type="range"]::-webkit-slider-thumb {
|
||||
pointer-events: auto;
|
||||
-webkit-appearance: none;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 50%;
|
||||
background: var(--accent);
|
||||
border: 2px solid #fff;
|
||||
box-shadow: 0 0 3px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.range-wrap input[type="range"]::-moz-range-thumb {
|
||||
pointer-events: auto;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 50%;
|
||||
background: var(--accent);
|
||||
border: 2px solid #fff;
|
||||
}
|
||||
|
||||
.range-track {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 6px;
|
||||
border-radius: 999px;
|
||||
background: #e3e7f1;
|
||||
transform: translateY(-50%);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.range-values {
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
margin-top: 4px;
|
||||
}
|
||||
.modal-content code {
|
||||
font-family: "JetBrains Mono", "SFMono-Regular", Consolas, monospace;
|
||||
background: #f6f8fc;
|
||||
padding: 1px 4px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
}
|
||||
@@ -0,0 +1,767 @@
|
||||
const DEFAULT_CTX = "default";
|
||||
const K_SIGMA = 1.0;
|
||||
const MIN_TOL = 0.25;
|
||||
const MODEL_COL_WIDTH = 180;
|
||||
const WINNER_COL_WIDTH = 120;
|
||||
|
||||
const state = {
|
||||
contexts: [],
|
||||
contextMap: new Map(),
|
||||
envs: [],
|
||||
backendOrder: [],
|
||||
columnWidths: {},
|
||||
filters: {
|
||||
search: "",
|
||||
quant: "",
|
||||
context: DEFAULT_CTX,
|
||||
backends: new Set(),
|
||||
sizeLo: null,
|
||||
sizeHi: null,
|
||||
},
|
||||
ui: {},
|
||||
sizeStats: { min: Infinity, max: -Infinity },
|
||||
draggingEnv: null,
|
||||
};
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
cacheUI();
|
||||
setupModals();
|
||||
try {
|
||||
const res = await fetch("results.json");
|
||||
const data = await res.json();
|
||||
prepareData(data?.runs || []);
|
||||
initializeControls();
|
||||
renderTables();
|
||||
} catch (err) {
|
||||
console.error("Failed to load results.json", err);
|
||||
state.ui.stats.textContent = "Failed to load results.json";
|
||||
}
|
||||
});
|
||||
|
||||
function cacheUI() {
|
||||
state.ui = {
|
||||
search: document.getElementById("filter-search"),
|
||||
quant: document.getElementById("filter-quant"),
|
||||
contextChips: document.getElementById("context-chips"),
|
||||
backendList: document.getElementById("backend-list"),
|
||||
backendAll: document.getElementById("backend-all"),
|
||||
backendNone: document.getElementById("backend-none"),
|
||||
sizeLo: document.getElementById("sizeLo"),
|
||||
sizeHi: document.getElementById("sizeHi"),
|
||||
sizeTrack: document.getElementById("sizeTrack"),
|
||||
sizeLoVal: document.getElementById("sizeLoVal"),
|
||||
sizeHiVal: document.getElementById("sizeHiVal"),
|
||||
stats: document.getElementById("stats-line"),
|
||||
resetBtn: document.getElementById("reset-layout"),
|
||||
tables: document.getElementById("tables"),
|
||||
hipblasModalOpen: document.getElementById("hipblas-modal-open"),
|
||||
hipblasModal: document.getElementById("hipblas-modal"),
|
||||
hipblasModalClose: document.getElementById("hipblas-modal-close"),
|
||||
rpcModalOpen: document.getElementById("rpc-modal-open"),
|
||||
rpcModal: document.getElementById("rpc-modal"),
|
||||
rpcModalClose: document.getElementById("rpc-modal-close"),
|
||||
rocwmmaModalOpen: document.getElementById("rocwmma-modal-open"),
|
||||
rocwmmaModal: document.getElementById("rocwmma-modal"),
|
||||
rocwmmaModalClose: document.getElementById("rocwmma-modal-close"),
|
||||
rocwmmaImprModalOpen: document.getElementById("rocwmma-impr-modal-open"),
|
||||
rocwmmaImprModal: document.getElementById("rocwmma-impr-modal"),
|
||||
rocwmmaImprModalClose: document.getElementById("rocwmma-impr-modal-close"),
|
||||
};
|
||||
}
|
||||
|
||||
function setupModals() {
|
||||
const modalConfigs = [
|
||||
{
|
||||
open: state.ui.hipblasModalOpen,
|
||||
modal: state.ui.hipblasModal,
|
||||
close: state.ui.hipblasModalClose,
|
||||
},
|
||||
{
|
||||
open: state.ui.rpcModalOpen,
|
||||
modal: state.ui.rpcModal,
|
||||
close: state.ui.rpcModalClose,
|
||||
},
|
||||
{
|
||||
open: state.ui.rocwmmaModalOpen,
|
||||
modal: state.ui.rocwmmaModal,
|
||||
close: state.ui.rocwmmaModalClose,
|
||||
},
|
||||
{
|
||||
open: state.ui.rocwmmaImprModalOpen,
|
||||
modal: state.ui.rocwmmaImprModal,
|
||||
close: state.ui.rocwmmaImprModalClose,
|
||||
},
|
||||
];
|
||||
|
||||
modalConfigs.forEach(({ open, modal, close }) => {
|
||||
if (!open || !modal) return;
|
||||
const openModal = () => modal.classList.remove("hidden");
|
||||
const closeModal = () => modal.classList.add("hidden");
|
||||
open.addEventListener("click", openModal);
|
||||
close?.addEventListener("click", closeModal);
|
||||
modal.addEventListener("click", (e) => {
|
||||
if (e.target === modal) closeModal();
|
||||
});
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape" && !modal.classList.contains("hidden")) {
|
||||
closeModal();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function prepareData(runs) {
|
||||
const contextMap = new Map();
|
||||
const envSet = new Set();
|
||||
const quantSet = new Set();
|
||||
|
||||
for (const run of runs) {
|
||||
const test = normalizeTest(run.test);
|
||||
if (!test || !run.env) continue;
|
||||
const contextKey = run.context || DEFAULT_CTX;
|
||||
const env = run.env;
|
||||
envSet.add(env);
|
||||
if (run.quant) quantSet.add(run.quant.toUpperCase());
|
||||
|
||||
const ctx = ensureContext(contextMap, contextKey, run.context_tokens);
|
||||
const testEntry = ensureTest(ctx, test.original);
|
||||
|
||||
const modelName = run.model_clean || run.model;
|
||||
const row = ensureModel(testEntry, modelName, run);
|
||||
row.backends[env] = {
|
||||
mean: typeof run.tps_mean === "number" ? run.tps_mean : null,
|
||||
std: typeof run.tps_std === "number" ? run.tps_std : null,
|
||||
error: Boolean(run.error),
|
||||
error_type: run.error_type || null,
|
||||
};
|
||||
}
|
||||
|
||||
state.contextMap = contextMap;
|
||||
state.contexts = [...contextMap.values()].sort((a, b) => {
|
||||
if (a.key === DEFAULT_CTX) return -1;
|
||||
if (b.key === DEFAULT_CTX) return 1;
|
||||
if (a.tokens && b.tokens) return a.tokens - b.tokens;
|
||||
if (a.tokens) return -1;
|
||||
if (b.tokens) return 1;
|
||||
return a.key.localeCompare(b.key);
|
||||
});
|
||||
state.envs = [...envSet].sort();
|
||||
state.backendOrder = [...state.envs];
|
||||
state.columnWidths = Object.fromEntries(state.envs.map((env) => [env, 120]));
|
||||
state.quantOptions = [...quantSet].sort();
|
||||
state.filters.context = state.contexts[0]?.key || DEFAULT_CTX;
|
||||
state.filters.backends = new Set(state.envs);
|
||||
}
|
||||
|
||||
function ensureContext(map, key, tokens) {
|
||||
if (!map.has(key)) {
|
||||
map.set(key, {
|
||||
key,
|
||||
label: formatContextLabel(key, tokens),
|
||||
tokens: tokens ?? null,
|
||||
tests: new Map(),
|
||||
});
|
||||
} else if (tokens && !map.get(key).tokens) {
|
||||
const ctx = map.get(key);
|
||||
ctx.tokens = tokens;
|
||||
ctx.label = formatContextLabel(key, tokens);
|
||||
}
|
||||
return map.get(key);
|
||||
}
|
||||
|
||||
function ensureTest(ctx, testName) {
|
||||
if (!ctx.tests.has(testName)) {
|
||||
ctx.tests.set(testName, {
|
||||
name: testName,
|
||||
models: new Map(),
|
||||
});
|
||||
}
|
||||
return ctx.tests.get(testName);
|
||||
}
|
||||
|
||||
function ensureModel(testEntry, modelName, run) {
|
||||
if (!testEntry.models.has(modelName)) {
|
||||
testEntry.models.set(modelName, {
|
||||
model: modelName,
|
||||
quant: (run.quant || "Unknown").toUpperCase(),
|
||||
sizeB: run.name_params_b ?? run.params_b ?? null,
|
||||
backends: {},
|
||||
isRpc: Boolean(run.rpc),
|
||||
search_blob: [modelName, run.quant, run.env, run.test]
|
||||
.filter(Boolean)
|
||||
.map((s) => s.toString().toLowerCase())
|
||||
.join(" "),
|
||||
});
|
||||
}
|
||||
const row = testEntry.models.get(modelName);
|
||||
const sizeCandidate = run.name_params_b ?? run.params_b;
|
||||
if (row.sizeB == null && typeof sizeCandidate === "number") {
|
||||
row.sizeB = sizeCandidate;
|
||||
}
|
||||
if (typeof row.sizeB === "number") {
|
||||
state.sizeStats.min = Math.min(state.sizeStats.min, row.sizeB);
|
||||
state.sizeStats.max = Math.max(state.sizeStats.max, row.sizeB);
|
||||
}
|
||||
if (run.rpc) {
|
||||
row.isRpc = true;
|
||||
if (!row.search_blob.includes("rpc")) {
|
||||
row.search_blob = `${row.search_blob} rpc`;
|
||||
}
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function initializeControls() {
|
||||
const { quant, contextChips, backendList, search, resetBtn, sizeLo, sizeHi } = state.ui;
|
||||
|
||||
quant.innerHTML = "";
|
||||
const anyOpt = document.createElement("option");
|
||||
anyOpt.value = "";
|
||||
anyOpt.textContent = "Any";
|
||||
quant.appendChild(anyOpt);
|
||||
state.quantOptions.forEach((q) => {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = q;
|
||||
opt.textContent = q;
|
||||
quant.appendChild(opt);
|
||||
});
|
||||
|
||||
contextChips.innerHTML = "";
|
||||
state.contexts.forEach((ctx) => {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "chip" + (ctx.key === state.filters.context ? " active" : "");
|
||||
btn.dataset.context = ctx.key;
|
||||
btn.textContent = ctx.label;
|
||||
contextChips.appendChild(btn);
|
||||
});
|
||||
|
||||
renderBackendList();
|
||||
setupSizeSlider();
|
||||
|
||||
search.addEventListener("input", (e) => {
|
||||
state.filters.search = (e.target.value || "").trim().toLowerCase();
|
||||
renderTables();
|
||||
});
|
||||
|
||||
quant.addEventListener("change", (e) => {
|
||||
state.filters.quant = e.target.value;
|
||||
renderTables();
|
||||
});
|
||||
|
||||
contextChips.addEventListener("click", (e) => {
|
||||
const btn = e.target.closest("button[data-context]");
|
||||
if (!btn) return;
|
||||
state.filters.context = btn.dataset.context;
|
||||
[...contextChips.querySelectorAll("button")].forEach((b) => b.classList.toggle("active", b === btn));
|
||||
renderTables();
|
||||
});
|
||||
|
||||
backendList.addEventListener("change", (e) => {
|
||||
const checkbox = e.target.closest("input[data-env]");
|
||||
if (!checkbox) return;
|
||||
const env = checkbox.dataset.env;
|
||||
if (checkbox.checked) {
|
||||
state.filters.backends.add(env);
|
||||
} else {
|
||||
state.filters.backends.delete(env);
|
||||
}
|
||||
renderTables();
|
||||
});
|
||||
|
||||
state.ui.backendAll.addEventListener("click", () => {
|
||||
state.filters.backends = new Set(state.envs);
|
||||
renderBackendList();
|
||||
renderTables();
|
||||
});
|
||||
|
||||
state.ui.backendNone.addEventListener("click", () => {
|
||||
state.filters.backends = new Set();
|
||||
renderBackendList();
|
||||
renderTables();
|
||||
});
|
||||
|
||||
sizeLo.addEventListener("input", () => updateSizeUI(true));
|
||||
sizeHi.addEventListener("input", () => updateSizeUI(true));
|
||||
|
||||
resetBtn.addEventListener("click", () => {
|
||||
state.filters.search = "";
|
||||
state.filters.quant = "";
|
||||
state.filters.context = state.contexts[0]?.key || DEFAULT_CTX;
|
||||
state.filters.backends = new Set(state.envs);
|
||||
search.value = "";
|
||||
quant.value = "";
|
||||
[...contextChips.querySelectorAll("button")].forEach((btn) =>
|
||||
btn.classList.toggle("active", btn.dataset.context === state.filters.context)
|
||||
);
|
||||
renderBackendList();
|
||||
setupSizeSlider();
|
||||
renderTables();
|
||||
});
|
||||
}
|
||||
|
||||
function renderBackendList() {
|
||||
const container = state.ui.backendList;
|
||||
container.innerHTML = "";
|
||||
state.backendOrder.forEach((env) => {
|
||||
const label = document.createElement("label");
|
||||
label.className = "backend-item";
|
||||
const checkbox = document.createElement("input");
|
||||
checkbox.type = "checkbox";
|
||||
checkbox.dataset.env = env;
|
||||
checkbox.checked = state.filters.backends.has(env);
|
||||
label.appendChild(checkbox);
|
||||
|
||||
const baseSpan = document.createElement("span");
|
||||
const { base, tags } = splitEnvName(env);
|
||||
baseSpan.textContent = base;
|
||||
label.appendChild(baseSpan);
|
||||
tags.forEach((tag) => {
|
||||
const pill = document.createElement("span");
|
||||
pill.className = "tag";
|
||||
pill.textContent = tag;
|
||||
const safeTag = tag.replace(/[^a-z0-9]+/gi, "-").toLowerCase();
|
||||
pill.classList.add(`tag-${safeTag}`);
|
||||
label.appendChild(pill);
|
||||
});
|
||||
|
||||
container.appendChild(label);
|
||||
});
|
||||
}
|
||||
|
||||
function setupSizeSlider() {
|
||||
const { sizeLo, sizeHi } = state.ui;
|
||||
const minRaw = state.sizeStats.min === Infinity ? 0 : Math.floor(state.sizeStats.min || 0);
|
||||
const maxRaw = state.sizeStats.max === -Infinity ? 0 : Math.ceil(state.sizeStats.max || 0);
|
||||
const minB = Math.max(0, minRaw);
|
||||
const maxB = Math.max(minB, maxRaw);
|
||||
|
||||
[sizeLo, sizeHi].forEach((inp) => {
|
||||
inp.min = minB;
|
||||
inp.max = maxB;
|
||||
inp.step = 1;
|
||||
});
|
||||
|
||||
sizeLo.value = minB;
|
||||
sizeHi.value = maxB;
|
||||
sizeLo.style.zIndex = 2;
|
||||
sizeHi.style.zIndex = 1;
|
||||
updateSizeUI(false);
|
||||
}
|
||||
|
||||
function updateSizeUI(triggerRender) {
|
||||
const { sizeLo, sizeHi, sizeLoVal, sizeHiVal, sizeTrack } = state.ui;
|
||||
if (+sizeLo.value > +sizeHi.value) {
|
||||
if (document.activeElement === sizeLo) {
|
||||
sizeHi.value = sizeLo.value;
|
||||
} else {
|
||||
sizeLo.value = sizeHi.value;
|
||||
}
|
||||
}
|
||||
sizeLo.style.zIndex = +sizeLo.value >= +sizeHi.max - 1 ? 4 : 2;
|
||||
sizeHi.style.zIndex = +sizeHi.value <= +sizeLo.min + 1 ? 3 : 1;
|
||||
state.filters.sizeLo = +sizeLo.value;
|
||||
state.filters.sizeHi = +sizeHi.value;
|
||||
sizeLoVal.textContent = formatSizeLabel(state.filters.sizeLo);
|
||||
sizeHiVal.textContent = formatSizeLabel(state.filters.sizeHi);
|
||||
const range = (sizeHi.max - sizeLo.min) || 1;
|
||||
const minB = +sizeLo.min;
|
||||
const start = ((state.filters.sizeLo - minB) / range) * 100;
|
||||
const end = ((state.filters.sizeHi - minB) / range) * 100;
|
||||
sizeTrack.style.background = `linear-gradient(to right, #e3e7f1 ${start}%, var(--accent) ${start}%, var(--accent) ${end}%, #e3e7f1 ${end}%)`;
|
||||
if (triggerRender) renderTables();
|
||||
}
|
||||
|
||||
function renderTables() {
|
||||
const ctx = state.contextMap.get(state.filters.context);
|
||||
if (!ctx) {
|
||||
state.ui.tables.innerHTML = "<p>No data for this context.</p>";
|
||||
state.ui.stats.textContent = "0 rows";
|
||||
return;
|
||||
}
|
||||
|
||||
const backendList = state.backendOrder.filter((env) => state.filters.backends.has(env));
|
||||
const tests = [...ctx.tests.values()].sort((a, b) => a.name.localeCompare(b.name));
|
||||
const frag = document.createDocumentFragment();
|
||||
let totalRows = 0;
|
||||
|
||||
for (const test of tests) {
|
||||
const models = filterModels(test.models);
|
||||
if (!models.length) continue;
|
||||
totalRows += models.length;
|
||||
const block = document.createElement("div");
|
||||
block.className = "test-block";
|
||||
const heading = document.createElement("h2");
|
||||
heading.textContent = `${test.name.toUpperCase()} — tokens/second`;
|
||||
block.appendChild(heading);
|
||||
|
||||
const tableWrap = document.createElement("div");
|
||||
tableWrap.className = "table-wrap";
|
||||
const scroller = document.createElement("div");
|
||||
scroller.className = "table-scroll";
|
||||
|
||||
const modelsWithWinners = models.map((model) => {
|
||||
const winners = computeWinners(model, backendList);
|
||||
return { ...model, _cachedWinners: winners };
|
||||
});
|
||||
|
||||
const table = buildSingleTable(modelsWithWinners, backendList);
|
||||
scroller.appendChild(table);
|
||||
tableWrap.appendChild(scroller);
|
||||
block.appendChild(tableWrap);
|
||||
setupResizeOverlay(scroller, backendList, table);
|
||||
frag.appendChild(block);
|
||||
}
|
||||
|
||||
state.ui.tables.innerHTML = "";
|
||||
if (frag.childNodes.length) {
|
||||
state.ui.tables.appendChild(frag);
|
||||
} else {
|
||||
state.ui.tables.innerHTML = "<p>No models match the current filters.</p>";
|
||||
}
|
||||
state.ui.stats.textContent = `Showing ${totalRows.toLocaleString()} model rows across ${backendList.length} backends`;
|
||||
}
|
||||
|
||||
function buildSingleTable(models, backendList) {
|
||||
const table = document.createElement("table");
|
||||
const colgroup = document.createElement("colgroup");
|
||||
const colModel = document.createElement("col");
|
||||
colModel.style.width = `${MODEL_COL_WIDTH}px`;
|
||||
colgroup.appendChild(colModel);
|
||||
const colWinner = document.createElement("col");
|
||||
colWinner.style.width = `${WINNER_COL_WIDTH}px`;
|
||||
colgroup.appendChild(colWinner);
|
||||
backendList.forEach((env) => {
|
||||
const col = document.createElement("col");
|
||||
col.style.width = `${state.columnWidths[env] || 120}px`;
|
||||
col.dataset.env = env;
|
||||
colgroup.appendChild(col);
|
||||
});
|
||||
table.appendChild(colgroup);
|
||||
|
||||
const thead = document.createElement("thead");
|
||||
const headRow = document.createElement("tr");
|
||||
headRow.appendChild(makeHeaderCell("Model", "model"));
|
||||
headRow.appendChild(makeHeaderCell("Winner", "winner"));
|
||||
backendList.forEach((env) => {
|
||||
const th = makeHeaderCell(env, "backend-header");
|
||||
attachHeaderInteractions(th, env);
|
||||
headRow.appendChild(th);
|
||||
});
|
||||
thead.appendChild(headRow);
|
||||
table.appendChild(thead);
|
||||
|
||||
const tbody = document.createElement("tbody");
|
||||
models.forEach((model) => {
|
||||
const tr = document.createElement("tr");
|
||||
const tdModel = document.createElement("td");
|
||||
tdModel.className = "model";
|
||||
const head = document.createElement("div");
|
||||
head.className = "model-head";
|
||||
const nameSpan = document.createElement("span");
|
||||
nameSpan.className = "model-name";
|
||||
nameSpan.textContent = model.model;
|
||||
head.appendChild(nameSpan);
|
||||
if (model.isRpc) {
|
||||
const pill = document.createElement("span");
|
||||
pill.className = "model-pill model-pill-rpc";
|
||||
pill.title = "Run executed via llama.cpp RPC across two servers";
|
||||
pill.textContent = "RPC · dual server";
|
||||
head.appendChild(pill);
|
||||
}
|
||||
tdModel.appendChild(head);
|
||||
const meta = document.createElement("div");
|
||||
meta.className = "meta";
|
||||
meta.textContent = `${model.quant} · ${formatSize(model.sizeB)}`;
|
||||
tdModel.appendChild(meta);
|
||||
|
||||
const actionWrap = document.createElement("div");
|
||||
actionWrap.className = "row-actions";
|
||||
const btnDesc = document.createElement("button");
|
||||
btnDesc.type = "button";
|
||||
btnDesc.className = "row-action-btn";
|
||||
btnDesc.textContent = "Sort ↓";
|
||||
btnDesc.addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
sortBackendsByModel(model, "desc");
|
||||
});
|
||||
const btnAsc = document.createElement("button");
|
||||
btnAsc.type = "button";
|
||||
btnAsc.className = "row-action-btn";
|
||||
btnAsc.textContent = "Sort ↑";
|
||||
btnAsc.addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
sortBackendsByModel(model, "asc");
|
||||
});
|
||||
actionWrap.appendChild(btnDesc);
|
||||
actionWrap.appendChild(btnAsc);
|
||||
tdModel.appendChild(actionWrap);
|
||||
tr.appendChild(tdModel);
|
||||
|
||||
const tdWinner = document.createElement("td");
|
||||
tdWinner.className = "winner";
|
||||
if (model._cachedWinners.length) {
|
||||
const wrap = document.createElement("div");
|
||||
wrap.className = "winner-list";
|
||||
wrap.innerHTML = model._cachedWinners.map((w) => `<span class="winner-pill">${w}</span>`).join("");
|
||||
tdWinner.appendChild(wrap);
|
||||
} else {
|
||||
tdWinner.innerHTML = `<span class="cell-empty">—</span>`;
|
||||
}
|
||||
|
||||
tr.appendChild(tdWinner);
|
||||
|
||||
backendList.forEach((env) => {
|
||||
const td = document.createElement("td");
|
||||
td.className = "data-cell";
|
||||
td.dataset.env = env;
|
||||
const cell = model.backends[env];
|
||||
if (!cell) {
|
||||
td.innerHTML = `<span class="cell-empty">—</span>`;
|
||||
} else if (cell.error || cell.mean == null) {
|
||||
td.innerHTML = `<span class="cell-error">⚠ ${cell.error_type || "error"}</span>`;
|
||||
} else {
|
||||
const isBest = model._cachedWinners.includes(env);
|
||||
if (isBest) td.classList.add("best");
|
||||
td.innerHTML = `<div class="measure">${cell.mean.toFixed(2)}</div><div class="std">± ${cell.std?.toFixed(2) ?? "—"}</div>`;
|
||||
}
|
||||
tr.appendChild(td);
|
||||
});
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
table.appendChild(tbody);
|
||||
return table;
|
||||
}
|
||||
|
||||
function makeHeaderCell(label, extra = "") {
|
||||
const th = document.createElement("th");
|
||||
th.textContent = label;
|
||||
if (extra) th.className = extra;
|
||||
return th;
|
||||
}
|
||||
|
||||
function attachHeaderInteractions(th, env) {
|
||||
const width = state.columnWidths[env] || 120;
|
||||
th.style.width = `${width}px`;
|
||||
th.style.minWidth = `${width}px`;
|
||||
th.draggable = true;
|
||||
th.addEventListener("dragstart", (e) => {
|
||||
state.draggingEnv = env;
|
||||
th.classList.add("dragging");
|
||||
e.dataTransfer.effectAllowed = "move";
|
||||
});
|
||||
th.addEventListener("dragend", () => {
|
||||
state.draggingEnv = null;
|
||||
th.classList.remove("dragging");
|
||||
document.querySelectorAll("th.backend-header.drop-target").forEach((el) => el.classList.remove("drop-target"));
|
||||
});
|
||||
th.addEventListener("dragover", (e) => {
|
||||
if (!state.draggingEnv || state.draggingEnv === env) return;
|
||||
e.preventDefault();
|
||||
th.classList.add("drop-target");
|
||||
});
|
||||
th.addEventListener("dragleave", () => th.classList.remove("drop-target"));
|
||||
th.addEventListener("drop", (e) => {
|
||||
if (!state.draggingEnv || state.draggingEnv === env) return;
|
||||
e.preventDefault();
|
||||
moveBackend(state.draggingEnv, env);
|
||||
th.classList.remove("drop-target");
|
||||
});
|
||||
|
||||
const handle = document.createElement("span");
|
||||
handle.className = "resize-handle";
|
||||
handle.addEventListener("mousedown", (e) => startResize(e, env));
|
||||
th.appendChild(handle);
|
||||
}
|
||||
|
||||
function moveBackend(from, to) {
|
||||
const order = state.backendOrder;
|
||||
const fromIdx = order.indexOf(from);
|
||||
const toIdx = order.indexOf(to);
|
||||
if (fromIdx === -1 || toIdx === -1) return;
|
||||
const [col] = order.splice(fromIdx, 1);
|
||||
order.splice(toIdx, 0, col);
|
||||
renderBackendList();
|
||||
renderTables();
|
||||
}
|
||||
|
||||
function filterModels(modelsMap) {
|
||||
const models = [];
|
||||
for (const model of modelsMap.values()) {
|
||||
if (state.filters.search && !model.search_blob.includes(state.filters.search)) continue;
|
||||
if (state.filters.quant && model.quant !== state.filters.quant) continue;
|
||||
if (model.sizeB != null) {
|
||||
if (state.filters.sizeLo != null && model.sizeB < state.filters.sizeLo - 1e-6) continue;
|
||||
if (state.filters.sizeHi != null && model.sizeB > state.filters.sizeHi + 1e-6) continue;
|
||||
}
|
||||
models.push(model);
|
||||
}
|
||||
models.sort((a, b) => a.model.localeCompare(b.model));
|
||||
return models;
|
||||
}
|
||||
|
||||
function computeWinners(model, backends) {
|
||||
const values = [];
|
||||
backends.forEach((env) => {
|
||||
const entry = model.backends[env];
|
||||
if (entry && !entry.error && typeof entry.mean === "number") {
|
||||
values.push({
|
||||
env,
|
||||
mean: entry.mean,
|
||||
std: typeof entry.std === "number" ? entry.std : 0,
|
||||
});
|
||||
}
|
||||
});
|
||||
if (!values.length) return [];
|
||||
let best = values[0];
|
||||
for (const v of values) if (v.mean > best.mean) best = v;
|
||||
const winners = [];
|
||||
for (const v of values) {
|
||||
const pooled = Math.sqrt((best.std || 0) ** 2 + (v.std || 0) ** 2);
|
||||
const tol = Math.max(MIN_TOL, K_SIGMA * pooled);
|
||||
if ((best.mean - v.mean) <= tol) winners.push(v.env);
|
||||
}
|
||||
return winners;
|
||||
}
|
||||
|
||||
function normalizeTest(name) {
|
||||
if (!name) return null;
|
||||
return { key: name.toLowerCase(), original: name };
|
||||
}
|
||||
|
||||
function formatContextLabel(key, tokens) {
|
||||
if (key === DEFAULT_CTX) return "Default window";
|
||||
if (tokens) return `ctx ${tokens.toLocaleString()}`;
|
||||
return key;
|
||||
}
|
||||
|
||||
function formatSize(size) {
|
||||
if (size == null) return "—";
|
||||
return `${Number(size).toFixed(1)}B`;
|
||||
}
|
||||
|
||||
function formatSizeLabel(size) {
|
||||
if (size >= 1000) return `${(size / 1000).toFixed(1)}kB`;
|
||||
return `${Math.round(size)}B`;
|
||||
}
|
||||
|
||||
function sortBackendsByModel(model, direction) {
|
||||
const dir = direction === "asc" ? 1 : -1;
|
||||
const order = [...state.backendOrder].sort((a, b) => {
|
||||
const va = backendValue(model.backends[a], direction);
|
||||
const vb = backendValue(model.backends[b], direction);
|
||||
if (va === vb) return a.localeCompare(b);
|
||||
return (va - vb) * dir;
|
||||
});
|
||||
state.backendOrder = order;
|
||||
renderBackendList();
|
||||
renderTables();
|
||||
}
|
||||
|
||||
function backendValue(entry, direction) {
|
||||
if (!entry || entry.error || typeof entry.mean !== "number") {
|
||||
return direction === "asc" ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
|
||||
}
|
||||
return entry.mean;
|
||||
}
|
||||
|
||||
function splitEnvName(env) {
|
||||
const canonical = env.replace(/_/g, ".");
|
||||
const tagRegex = /-(rocwmma-improved|rocwmma|improved|hblt0)/gi;
|
||||
const tags = [];
|
||||
let match;
|
||||
while ((match = tagRegex.exec(canonical)) !== null) {
|
||||
tags.push(match[1].toLowerCase());
|
||||
}
|
||||
const base = canonical.replace(tagRegex, "");
|
||||
return { base, tags };
|
||||
}
|
||||
|
||||
function startResize(event, env) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
const column = state.columnWidths[env] || 120;
|
||||
const startX = event.clientX;
|
||||
const shellRect = state.ui.tables.getBoundingClientRect();
|
||||
const guide = document.createElement("div");
|
||||
guide.className = "resize-line";
|
||||
guide.style.position = "fixed";
|
||||
guide.style.top = `${shellRect.top}px`;
|
||||
guide.style.bottom = `${window.innerHeight - shellRect.bottom}px`;
|
||||
guide.style.left = `${startX}px`;
|
||||
guide.style.width = "2px";
|
||||
guide.style.background = "var(--accent)";
|
||||
guide.style.zIndex = "10";
|
||||
document.body.appendChild(guide);
|
||||
let nextWidth = column;
|
||||
|
||||
const onMove = (e) => {
|
||||
const delta = e.clientX - startX;
|
||||
nextWidth = Math.max(80, column + delta);
|
||||
guide.style.left = `${e.clientX}px`;
|
||||
};
|
||||
|
||||
const onUp = () => {
|
||||
document.removeEventListener("mousemove", onMove);
|
||||
document.removeEventListener("mouseup", onUp);
|
||||
guide.remove();
|
||||
state.columnWidths[env] = nextWidth;
|
||||
renderTables();
|
||||
};
|
||||
|
||||
document.addEventListener("mousemove", onMove);
|
||||
document.addEventListener("mouseup", onUp);
|
||||
}
|
||||
|
||||
function setupResizeOverlay(tableWrap, backendList, table) {
|
||||
let overlay = tableWrap.querySelector(".resize-overlay");
|
||||
if (!overlay) {
|
||||
overlay = document.createElement("div");
|
||||
overlay.className = "resize-overlay";
|
||||
tableWrap.appendChild(overlay);
|
||||
} else {
|
||||
overlay.innerHTML = "";
|
||||
}
|
||||
|
||||
overlay.style.width = `${tableWrap.clientWidth}px`;
|
||||
overlay.style.height = `${table.offsetHeight}px`;
|
||||
|
||||
const bars = [];
|
||||
let offset = MODEL_COL_WIDTH + WINNER_COL_WIDTH;
|
||||
backendList.forEach((env) => {
|
||||
const width = state.columnWidths[env] || 120;
|
||||
const bar = document.createElement("div");
|
||||
bar.className = "resize-bar";
|
||||
bar.dataset.env = env;
|
||||
bar.addEventListener("mousedown", (e) => startResize(e, env));
|
||||
overlay.appendChild(bar);
|
||||
bars.push({ bar, offset, width, env });
|
||||
offset += width;
|
||||
});
|
||||
|
||||
const positionBars = () => {
|
||||
bars.forEach(({ bar, offset, width }) => {
|
||||
const left = offset + width - 3 - tableWrap.scrollLeft;
|
||||
bar.style.left = `${left}px`;
|
||||
});
|
||||
};
|
||||
positionBars();
|
||||
|
||||
if (tableWrap._overlayScroll) {
|
||||
tableWrap.removeEventListener("scroll", tableWrap._overlayScroll);
|
||||
}
|
||||
const onScroll = () => positionBars();
|
||||
tableWrap.addEventListener("scroll", onScroll);
|
||||
tableWrap._overlayScroll = onScroll;
|
||||
|
||||
if (tableWrap._overlayResize) {
|
||||
tableWrap._overlayResize.disconnect();
|
||||
}
|
||||
const resizeObserver = new ResizeObserver(() => {
|
||||
overlay.style.width = `${tableWrap.clientWidth}px`;
|
||||
overlay.style.height = `${table.offsetHeight}px`;
|
||||
positionBars();
|
||||
});
|
||||
resizeObserver.observe(tableWrap);
|
||||
tableWrap._overlayResize = resizeObserver;
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
# AMD Strix Halo — llama.cpp Toolboxes (Benchmarks)
|
||||
|
||||
**Interactive results:** https://kyuz0.github.io/amd-strix-halo-toolboxes/
|
||||
|
||||
## Table of Contents
|
||||
- [Benchmark methodology](#benchmark-methodology)
|
||||
- [Summary of current dataset (Flash Attention ON)](#summary-of-current-dataset-flash-attention-on)
|
||||
- [Placement counts](#placement-counts)
|
||||
- [Pairwise head-to-head wins](#pairwise-head-to-head-wins)
|
||||
- [Average ranks](#average-ranks)
|
||||
- [Analyses by feature](#analyses-by-feature)
|
||||
- [Impact of Flash Attention](#impact-of-flash-attention)
|
||||
- [Impact of ROCWMMA](#impact-of-rocwmma)
|
||||
- [Impact of hipBLASLt](#impact-of-hipblaslt)
|
||||
- [Vulkan: AMDVLK vs RADV](#vulkan-amdvlk-vs-radv)
|
||||
- [Recommendations](#recommendations)
|
||||
- [Winner calculation](#winner-calculation)
|
||||
|
||||
---
|
||||
|
||||
## Benchmark methodology
|
||||
|
||||
- **pp512** — prompt processing throughput (tokens/sec, prefill)
|
||||
- **tg128** — token generation throughput (tokens/sec, interactive)
|
||||
- Each backend tested twice per model: `-fa 0` and `-fa 1`
|
||||
- Winners per model/test are **margin-aware**; multiple winners are possible when mean±σ overlap
|
||||
- Built from the same llama.cpp commit for consistency
|
||||
|
||||
**Backends in this dataset:** ROCm 7 RC + ROCWMMA + hipBLASLt, ROCm 7 RC (hipBLASLt), ROCm 7 RC (hipBLASLt OFF), ROCm 7 RC + ROCWMMA (hipBLASLt OFF), ROCm 6.4.4 (hipBLASLt), ROCm 6.4.4 (hipBLASLt OFF), ROCm 6.4.4 + ROCWMMA (hipBLASLt), ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF), Vulkan AMDVLK, Vulkan RADV
|
||||
|
||||
**ROCm 7 hipBLASLt policy:** Toolboxes ship with **hipBLASLt enabled** by default (`ROCBLAS_USE_HIPBLASLT=1`). The benchmark script also runs **hipBLASLt OFF** variants (`-hblt0`) to measure its effect.
|
||||
|
||||
---
|
||||
|
||||
## Summary of current dataset (Flash Attention ON)
|
||||
|
||||
### Placement counts
|
||||
**Prompt Processing (pp512)**
|
||||
| Backend | 1st | 2nd | 3rd |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| ROCm 6.4.4 (hipBLASLt) | 6 | 2 | 2 |
|
||||
| Vulkan AMDVLK | 6 | 1 | 0 |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | 3 | 2 | 3 |
|
||||
| Vulkan RADV | 1 | 2 | 0 |
|
||||
| ROCm 7 RC (hipBLASLt) | 1 | 1 | 1 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 0 | 5 | 4 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt) | 0 | 4 | 2 |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | 0 | 0 | 2 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt | 0 | 0 | 3 |
|
||||
|
||||
**Token Generation (tg128)**
|
||||
| Backend | 1st | 2nd | 3rd |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Vulkan RADV | 10 | 1 | 2 |
|
||||
| Vulkan AMDVLK | 3 | 10 | 0 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 2 | 3 | 7 |
|
||||
| ROCm 6.4.4 (hipBLASLt) | 1 | 4 | 3 |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | 1 | 3 | 5 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt) | 1 | 2 | 6 |
|
||||
| ROCm 7 RC (hipBLASLt) | 1 | 0 | 1 |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | 0 | 1 | 1 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt | 0 | 1 | 1 |
|
||||
| ROCm 7 RC + ROCWMMA (hipBLASLt OFF) | 0 | 1 | 1 |
|
||||
|
||||
### Pairwise head-to-head wins
|
||||
For any model+quant where both backends succeeded, this counts who was faster (ties when equal).
|
||||
| Comparison | Test | A wins | B wins | Ties | Total |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt vs Vulkan AMDVLK | pp512 | 9 | 7 | 0 | 16 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt vs Vulkan AMDVLK | tg128 | 2 | 14 | 0 | 16 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt vs Vulkan RADV | pp512 | 14 | 3 | 0 | 17 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt vs Vulkan RADV | tg128 | 4 | 12 | 1 | 17 |
|
||||
| Vulkan AMDVLK vs Vulkan RADV | pp512 | 12 | 4 | 0 | 16 |
|
||||
| Vulkan AMDVLK vs Vulkan RADV | tg128 | 5 | 11 | 0 | 16 |
|
||||
|
||||
### Average ranks
|
||||
**Prompt Processing (pp512)**
|
||||
| Backend | Avg Rank (↓ is better) |
|
||||
| --- | ---: |
|
||||
| Vulkan AMDVLK | 1.14 |
|
||||
| ROCm 6.4.4 (hipBLASLt) | 1.6 |
|
||||
| Vulkan RADV | 1.67 |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | 2.0 |
|
||||
| ROCm 7 RC (hipBLASLt) | 2.0 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt) | 2.33 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 2.44 |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | 3.0 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt | 3.0 |
|
||||
|
||||
**Token Generation (tg128)**
|
||||
| Backend | Avg Rank (↓ is better) |
|
||||
| --- | ---: |
|
||||
| Vulkan RADV | 1.38 |
|
||||
| Vulkan AMDVLK | 1.77 |
|
||||
| ROCm 7 RC (hipBLASLt) | 2.0 |
|
||||
| ROCm 6.4.4 (hipBLASLt) | 2.25 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 2.42 |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | 2.44 |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt | 2.5 |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | 2.5 |
|
||||
| ROCm 7 RC + ROCWMMA (hipBLASLt OFF) | 2.5 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt) | 2.56 |
|
||||
|
||||
---
|
||||
|
||||
## Analyses by feature
|
||||
|
||||
### Impact of Flash Attention
|
||||
Median % change when **Flash Attention ON vs OFF**, paired by model+quant, per backend:
|
||||
| Backend | pp512 Δ% (median, min..max, n) | tg128 Δ% (median, min..max, n) |
|
||||
| --- | --- | --- |
|
||||
| ROCm 7 RC + ROCWMMA + hipBLASLt | 11.4% (4.2..34.1), n=17 | -0.5% (-8.8..0.8), n=17 |
|
||||
| ROCm 7 RC (hipBLASLt) | 11.7% (-23.0..25.6), n=14 | -1.1% (-8.7..1.0), n=14 |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | 6.8% (2.1..18.4), n=15 | -0.8% (-9.0..0.5), n=15 |
|
||||
| ROCm 7 RC + ROCWMMA (hipBLASLt OFF) | 6.3% (-5.5..17.4), n=16 | -0.8% (-15.1..0.6), n=16 |
|
||||
| ROCm 6.4.4 (hipBLASLt) | 8.3% (5.6..20.8), n=17 | 0.8% (-3.0..2.6), n=17 |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | 7.2% (-0.5..19.5), n=17 | 1.1% (-2.9..2.7), n=17 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt) | 7.1% (5.0..19.9), n=17 | 0.9% (-2.8..2.8), n=17 |
|
||||
| ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 6.5% (2.7..18.6), n=17 | 1.1% (-2.7..3.4), n=17 |
|
||||
| Vulkan AMDVLK | 1.3% (-10.8..27.8), n=16 | -1.2% (-6.8..0.1), n=16 |
|
||||
| Vulkan RADV | 4.8% (-0.5..20.1), n=17 | -0.1% (-2.1..2.0), n=17 |
|
||||
|
||||
### Impact of ROCWMMA
|
||||
| Context | Test | Compared Envs | Pairs | Median Δ% |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| ROCm 7 RC (hipBLASLt) | pp512 | ROCm 7 RC + ROCWMMA + hipBLASLt vs ROCm 7 RC (hipBLASLt) | 15 | -0.0% |
|
||||
| ROCm 7 RC (hipBLASLt) | tg128 | ROCm 7 RC + ROCWMMA + hipBLASLt vs ROCm 7 RC (hipBLASLt) | 15 | 0.0% |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | pp512 | ROCm 7 RC + ROCWMMA (hipBLASLt OFF) vs ROCm 7 RC (hipBLASLt OFF) | 17 | -0.2% |
|
||||
| ROCm 7 RC (hipBLASLt OFF) | tg128 | ROCm 7 RC + ROCWMMA (hipBLASLt OFF) vs ROCm 7 RC (hipBLASLt OFF) | 17 | 0.0% |
|
||||
| ROCm 6.4.4 (hipBLASLt) | pp512 | ROCm 6.4.4 + ROCWMMA (hipBLASLt) vs ROCm 6.4.4 (hipBLASLt) | 17 | -0.4% |
|
||||
| ROCm 6.4.4 (hipBLASLt) | tg128 | ROCm 6.4.4 + ROCWMMA (hipBLASLt) vs ROCm 6.4.4 (hipBLASLt) | 17 | 0.0% |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | pp512 | ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) vs ROCm 6.4.4 (hipBLASLt OFF) | 17 | -0.5% |
|
||||
| ROCm 6.4.4 (hipBLASLt OFF) | tg128 | ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) vs ROCm 6.4.4 (hipBLASLt OFF) | 17 | -0.1% |
|
||||
|
||||
### Impact of hipBLASLt
|
||||
| Context | Test | Compared Envs | Pairs | Median Δ% |
|
||||
| --- | --- | --- | ---: | ---: |
|
||||
| ROCm 7 RC (no ROCWMMA) | pp512 | ROCm 7 RC (hipBLASLt) vs ROCm 7 RC (hipBLASLt OFF) | 15 | -0.2% |
|
||||
| ROCm 7 RC (no ROCWMMA) | tg128 | ROCm 7 RC (hipBLASLt) vs ROCm 7 RC (hipBLASLt OFF) | 15 | 0.0% |
|
||||
| ROCm 7 RC + ROCWMMA | pp512 | ROCm 7 RC + ROCWMMA + hipBLASLt vs ROCm 7 RC + ROCWMMA (hipBLASLt OFF) | 17 | -0.1% |
|
||||
| ROCm 7 RC + ROCWMMA | tg128 | ROCm 7 RC + ROCWMMA + hipBLASLt vs ROCm 7 RC + ROCWMMA (hipBLASLt OFF) | 17 | 0.0% |
|
||||
| ROCm 6.4.4 (no ROCWMMA) | pp512 | ROCm 6.4.4 (hipBLASLt) vs ROCm 6.4.4 (hipBLASLt OFF) | 17 | 0.0% |
|
||||
| ROCm 6.4.4 (no ROCWMMA) | tg128 | ROCm 6.4.4 (hipBLASLt) vs ROCm 6.4.4 (hipBLASLt OFF) | 17 | 0.0% |
|
||||
| ROCm 6.4.4 + ROCWMMA | pp512 | ROCm 6.4.4 + ROCWMMA (hipBLASLt) vs ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 17 | -0.3% |
|
||||
| ROCm 6.4.4 + ROCWMMA | tg128 | ROCm 6.4.4 + ROCWMMA (hipBLASLt) vs ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF) | 17 | 0.0% |
|
||||
|
||||
### Vulkan: AMDVLK vs RADV
|
||||
Head-to-head wins with selected Flash Attention filter:
|
||||
| Test | AMDVLK wins | RADV wins | Ties | Total |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| pp512 | 12 | 4 | 0 | 16 |
|
||||
| tg128 | 5 | 11 | 0 | 16 |
|
||||
|
||||
---
|
||||
|
||||
## Recommendations
|
||||
- **Fastest prompt processing:** Vulkan AMDVLK, ROCm 6.4.4 (hipBLASLt) (most 1st-place finishes with selected Flash Attention filter).
|
||||
- **Fastest token generation:** Vulkan RADV (most 1st-place finishes with selected Flash Attention filter).
|
||||
- **Balanced choice:** Vulkan AMDVLK (consistently near the top across PP/TG).
|
||||
|
||||
---
|
||||
|
||||
## Winner calculation
|
||||
A backend is counted as a winner if its mean throughput is within the best backend’s pooled ± error margin for that model/test type. This treats results within measurement noise as ties instead of false losses.
|
||||
@@ -0,0 +1,75 @@
|
||||
|
||||
# Building Containers Locally
|
||||
|
||||
If you want to build or customize the toolbox containers yourself (rather than using the pre-built Docker Hub images), this guide explains the process. Local builds are useful if you want to:
|
||||
|
||||
* Use a patched or forked version of llama.cpp
|
||||
* Add additional tools or libraries
|
||||
* Change the Fedora base image (Rawhide vs. stable)
|
||||
* Audit every installed dependency
|
||||
|
||||
---
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
* **Podman** (recommended on Fedora) or **Docker** (also fine)
|
||||
|
||||
---
|
||||
|
||||
## 2. Build an Image
|
||||
|
||||
Each backend has its own subdirectory and Dockerfile in `toolboxes/`.
|
||||
|
||||
**Example: Build the Vulkan RADV toolbox image**
|
||||
|
||||
```sh
|
||||
cd toolboxes
|
||||
podman build --no-cache -t llama-vulkan-radv -f Dockerfile.vulkan-radv .
|
||||
```
|
||||
|
||||
**Example: Build the ROCm 6.4.2 toolbox image**
|
||||
|
||||
```sh
|
||||
cd toolboxes
|
||||
podman build --no-cache -t llama-rocm-6.4.2 -f Dockerfile.rocm-6.4.2 .
|
||||
```
|
||||
|
||||
> You can use `docker build` if you prefer Docker.
|
||||
|
||||
---
|
||||
|
||||
## 3. Customizing the Build
|
||||
|
||||
* **llama.cpp version**: Change the `git clone` or `git checkout` line in the Dockerfile.
|
||||
* **Extra dependencies**: Add them to the Dockerfile as needed.
|
||||
* **Other customizations**: Install tools, patch scripts, or swap to a different base image.
|
||||
|
||||
---
|
||||
|
||||
## 4. Using the Custom Image with Toolbx
|
||||
|
||||
Create a new toolbox using your freshly built image:
|
||||
|
||||
```sh
|
||||
toolbox create llama-vulkan-radv --image localhost/llama-vulkan-radv \
|
||||
-- --device /dev/dri --group-add video --security-opt seccomp=unconfined
|
||||
```
|
||||
|
||||
Replace the backend/image name and device/group options as needed (see main README Section 2.1).
|
||||
|
||||
---
|
||||
|
||||
## 5. Troubleshooting
|
||||
|
||||
* **Build fails (ROCm images especially):** Try building with more memory or swap.
|
||||
* **Toolbox can't access GPU:** Make sure you pass the correct device/group options.
|
||||
|
||||
---
|
||||
|
||||
## 6. References
|
||||
|
||||
* [Fedora Toolbox Documentation](https://docs.fedoraproject.org/en-US/fedora-silverblue/toolbox/)
|
||||
* [Podman Build Reference](https://docs.podman.io/en/latest/markdown/podman-build.1.html)
|
||||
* [Docker Build Reference](https://docs.docker.com/engine/reference/commandline/build/)
|
||||
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
## How to use docker-compose instead of toolbox
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Vulkan AMDVLK](#1-vulkanamdvlk)
|
||||
2. [ROCm-6.4.4+ROCWMMA](#2-rocm-644-rocwmma)
|
||||
|
||||
## 1. Vulkan(AMDVLK)
|
||||
|
||||
1. Select applicable backend Dockerfile from repo. Example:
|
||||
https://github.com/kyuz0/amd-strix-halo-toolboxes/blob/main/toolboxes/Dockerfile.vulkan-amdvlk
|
||||
|
||||
2. In the build file, change shell command to:
|
||||
|
||||
|
||||
```
|
||||
# shell
|
||||
CMD ["/bin/bash", "-c", "llama-server --host $HOST --port $PORT -c $CONTEXT_LENGTH --temp $TEMPERATURE --jinja --no-mmap -ngl $NGL -fa $FA -m $MODEL_PATH"]
|
||||
```
|
||||
|
||||
3. Build container with:
|
||||
|
||||
```
|
||||
docker build -f Dockerfile.vulkan-amdvlk -t vulkan-amdvlk:1.0 .
|
||||
```
|
||||
|
||||
4. Download your model files to a directory. We will mount this from the container. I use:
|
||||
|
||||
```
|
||||
/mnt/models
|
||||
```
|
||||
|
||||
5. Create your docker compose, using this template. Change the ports and paths as needed.
|
||||
|
||||
```
|
||||
services:
|
||||
gpt-oss-120b:
|
||||
container_name: gpt-oss-120b
|
||||
image: vulkan-amdvlk:1.0
|
||||
ports:
|
||||
- "8069:8069"
|
||||
volumes:
|
||||
- /mnt/models:/mnt/models
|
||||
devices:
|
||||
- "/dev/dri:/dev/dri"
|
||||
privileged: true
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- HOST=0.0.0.0
|
||||
- PORT=8069
|
||||
- CONTEXT_LENGTH=120000
|
||||
- TEMPERATURE=0.0
|
||||
- MODEL_PATH=/mnt/models/gpt-oss-120b-UD-Q4_K_XL/gpt-oss-120b-UD-Q4_K_XL-00001-of-00002.gguf
|
||||
- NGL=999
|
||||
- FA=on
|
||||
```
|
||||
|
||||
6. Start as usual.
|
||||
|
||||
```
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 2. ROCm-6.4.4-ROCWMMA
|
||||
|
||||
1. Select applicable backend Dockerfile from repo. Example:
|
||||
https://github.com/kyuz0/amd-strix-halo-toolboxes/blob/main/toolboxes/Dockerfile.rocm-6.4.4-rocwmma
|
||||
|
||||
3. In the build file, change shell command to:
|
||||
|
||||
|
||||
```
|
||||
# shell
|
||||
CMD ["/bin/bash", "-c", "llama-server --host $HOST --port $PORT -c $CONTEXT_LENGTH --temp $TEMPERATURE --jinja --no-mmap -ngl $NGL -fa $FA -m $MODEL_PATH"]
|
||||
```
|
||||
|
||||
3. Build container with:
|
||||
|
||||
```
|
||||
docker build -f Dockerfile.rocm-6.4.4-rocwmma -t rocm-6.4.4-rocwmma:1.0 .
|
||||
```
|
||||
|
||||
4. Download your model files to a directory. We will mount this from the container. I use:
|
||||
|
||||
```
|
||||
/mnt/models
|
||||
```
|
||||
|
||||
5. Create your docker compose, using this template. Change the ports and paths as needed.
|
||||
|
||||
```
|
||||
services:
|
||||
gpt-oss-120b:
|
||||
container_name: gpt-oss-120b
|
||||
image: rocm-6.4.4-rocwmma:1.0
|
||||
ports:
|
||||
- "8069:8069"
|
||||
volumes:
|
||||
- /mnt/models:/mnt/models
|
||||
devices:
|
||||
- "/dev/dri:/dev/dri"
|
||||
- "/dev/kfd:/dev/kfd"
|
||||
privileged: true
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- HOST=0.0.0.0
|
||||
- PORT=8069
|
||||
- CONTEXT_LENGTH=120000
|
||||
- TEMPERATURE=0.0
|
||||
- MODEL_PATH=/mnt/models/gpt-oss-120b-UD-Q4_K_XL/gpt-oss-120b-UD-Q4_K_XL-00001-of-00002.gguf
|
||||
- NGL=999
|
||||
- FA=on
|
||||
```
|
||||
|
||||
6. Start as usual.
|
||||
|
||||
```
|
||||
docker compose up -d
|
||||
```
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>AMD Radeon AI PRO R9700 — Backend Benchmarks (Grid View)</title>
|
||||
<link rel="stylesheet" href="assets/index2.css">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<header>
|
||||
<h1>AMD Radeon AI PRO R9700 — Benchmark Grid</h1>
|
||||
<p>AMD Radeon AI PRO R9700 · 32GB vRAM</p>
|
||||
<p>Fedora 43 · Linux 6.17.8-300.fc43.x86_64 · llama.cpp build 1c398dc9e (7034)</p>
|
||||
<p>Benchmarks captured 14 Nov 2025 · Repo: <a href="https://github.com/kyuz0/amd-r9700-toolboxes"
|
||||
target="_blank" rel="noreferrer">kyuz0/amd-r9700-toolboxes</a></p>
|
||||
<div class="legend">
|
||||
<label>Legend</label>
|
||||
<div class="legend-pills">
|
||||
<button id="hipblas-modal-open" type="button" class="chip small legend-pill legend-pill-default">
|
||||
hipBLASLt vs hblt0
|
||||
</button>
|
||||
<button id="rpc-modal-open" type="button" class="chip small legend-pill legend-pill-rpc">
|
||||
RPC · dual server
|
||||
</button>
|
||||
<button id="rocwmma-modal-open" type="button" class="chip small legend-pill legend-pill-rocwmma">
|
||||
rocWMMA
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="controls">
|
||||
<div class="control">
|
||||
<label for="filter-search">Search models</label>
|
||||
<input id="filter-search" type="text" placeholder="e.g. llama, qwen, 30B…">
|
||||
</div>
|
||||
<div class="control">
|
||||
<label for="filter-quant">Quant</label>
|
||||
<select id="filter-quant">
|
||||
<option value="">Any</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="control grow slider-block">
|
||||
<label>Context windows</label>
|
||||
<div id="context-chips" class="chip-row tight"></div>
|
||||
</div>
|
||||
<div class="control grow slider-block">
|
||||
<label>Model params (B)</label>
|
||||
<div class="range-wrap">
|
||||
<input type="range" id="sizeLo" step="1">
|
||||
<input type="range" id="sizeHi" step="1">
|
||||
<div class="range-track" id="sizeTrack"></div>
|
||||
</div>
|
||||
<div class="range-values">
|
||||
<span id="sizeLoVal">0B</span> – <span id="sizeHiVal">0B</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel compact">
|
||||
<div class="panel-split">
|
||||
<div class="backend-header">
|
||||
<div class="backend-label">
|
||||
<label>Backends</label>
|
||||
<div class="backend-actions">
|
||||
<button type="button" id="backend-all" class="chip small">All</button>
|
||||
<button type="button" id="backend-none" class="chip small">None</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="backend-list" class="backend-list"></div>
|
||||
</div>
|
||||
<div class="stats-box">
|
||||
<div class="stat-line" id="stats-line">Loading…</div>
|
||||
<button id="reset-layout" type="button" class="chip small">Reset filters</button>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel compact" id="tables-panel">
|
||||
<div id="tables"></div>
|
||||
</section>
|
||||
|
||||
<div id="hipblas-modal" class="modal hidden" role="dialog" aria-modal="true" aria-labelledby="hipblas-title">
|
||||
<div class="modal-content">
|
||||
<button id="hipblas-modal-close" class="modal-close" aria-label="Close dialog">×</button>
|
||||
<h2 id="hipblas-title">hipBLASLt & hblt0 explained</h2>
|
||||
<p>The ROCm toolboxes ship with <code>ROCBLAS_USE_HIPBLASLT=1</code> by default. This forces rocBLAS to
|
||||
prefer
|
||||
the hipBLASLt kernel library, which historically delivered the best throughput on gfx1201 (R9700).</p>
|
||||
<p>Rows tagged with <code>__hblt0</code> were re-run with <code>ROCBLAS_USE_HIPBLASLT=0</code>, letting
|
||||
rocBLAS
|
||||
auto-select between hipBLASLt, Tensile, or other kernel providers. These runs show how performance
|
||||
shifts when
|
||||
the tuned hipBLASLt path is disabled.</p>
|
||||
<p>hipBLASLt is AMD's LT (low-level tuned) matmul backend, optimized for transformer workloads. Disabling it
|
||||
can
|
||||
expose regressions or improvements depending on driver versions, so both configurations are published
|
||||
for
|
||||
comparison.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="rpc-modal" class="modal hidden" role="dialog" aria-modal="true" aria-labelledby="rpc-title">
|
||||
<div class="modal-content">
|
||||
<button id="rpc-modal-close" class="modal-close" aria-label="Close dialog">×</button>
|
||||
<h2 id="rpc-title">RPC · dual server</h2>
|
||||
<p>These results were produced with two R9700 systems (each 32 GB)
|
||||
connected over 5 Gbps Ethernet. One runs <code>rpc-server</code> from llama.cpp; the other runs
|
||||
<code>llama-bench --rpc</code>.
|
||||
</p>
|
||||
<p>This setup allows distributed inference, splitting large GGUF models across both machines. The metric
|
||||
shows what
|
||||
you can expect when latency is limited by the network and the workload is balanced between two RPC
|
||||
participants.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="rocwmma-modal" class="modal hidden" role="dialog" aria-modal="true" aria-labelledby="rocwmma-title">
|
||||
<div class="modal-content">
|
||||
<button id="rocwmma-modal-close" class="modal-close" aria-label="Close dialog">×</button>
|
||||
<h2 id="rocwmma-title">rocWMMA variants</h2>
|
||||
<p>Backends labeled <code>-rocwmma</code> are rebuilt with AMD's rocWMMA library, which unlocks matrix
|
||||
multiply
|
||||
pipelines accelerated via wave matrix multiply-accumulate (WMMA) instructions.</p>
|
||||
<p>rocWMMA kernels can significantly accelerate BF16/F16 workloads on RDNA3 but may trade stability or
|
||||
memory
|
||||
usage; comparing plain toolboxes against <code>-rocwmma</code> ones highlights the benefit or cost.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<script src="assets/index2.js" type="module"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
+45422
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,89 @@
|
||||
---
|
||||
|
||||
## docs/vram-estimator.md
|
||||
|
||||
---
|
||||
|
||||
# 1. Memory Planning with `gguf-vram-estimator.py`
|
||||
|
||||
Estimating memory requirements is critical when running large models on Strix Halo (or any GPU with limited RAM). It's not enough to check just the model file size: context length and runtime overheads matter.
|
||||
|
||||
This repo provides a tool, **`gguf-vram-estimator.py`**, which reads a `.gguf` model and prints the estimated VRAM needed for different context sizes.
|
||||
|
||||
**Why?**
|
||||
|
||||
* Helps decide what fits on 32GB, 64GB, 128GB, etc—especially with multi-shard models or large quantized files.
|
||||
|
||||
---
|
||||
|
||||
## 2. Usage
|
||||
|
||||
Make sure you have the estimator script (in `tools/`):
|
||||
|
||||
```sh
|
||||
gguf-vram-estimator.py <path-to-model.gguf>
|
||||
```
|
||||
|
||||
* Supply one or more context lengths to get the corresponding VRAM footprint.
|
||||
* Handles multi-shard and single-shard models.
|
||||
|
||||
---
|
||||
|
||||
## 3. Examples
|
||||
|
||||
### 3.1 Llama-4-Scout 17B Q4\_K\_XL, up to 1M tokens
|
||||
|
||||
```
|
||||
$ gguf-vram-estimator.py models/llama-4-scout-17b-16e/Q4_K_XL/Llama-4-Scout-17B-16E-Instruct-UD-Q4_K_XL-00001-of-00002.gguf --contexts 4096 32768 1048576
|
||||
|
||||
--- Model 'Llama-4-Scout-17B-16E-Instruct' ---
|
||||
Max Context: 10,485,760 tokens
|
||||
Model Size: 57.74 GiB
|
||||
Incl. Overhead: 2.00 GiB
|
||||
|
||||
--- Memory Footprint Estimation ---
|
||||
Context Size | Context Memory | Est. Total VRAM
|
||||
---------------------------------------------------
|
||||
4,096 | 1.88 GiB | 61.62 GiB
|
||||
32,768 | 15.06 GiB | 74.80 GiB
|
||||
1,048,576 | 49.12 GiB | 108.87 GiB
|
||||
```
|
||||
|
||||
* **Takeaway:**
|
||||
|
||||
* Q4\_K quantization allows for a huge context in 128GB, but *processing 1M tokens will be extremely slow* (see benchmark: 200 tokens/sec prompt processing ⇒ almost 1.5 hours for a full 1M context fill).
|
||||
|
||||
---
|
||||
|
||||
### 3.2 Qwen3-235B Q3\_K XL, high context
|
||||
|
||||
```
|
||||
$ gguf-vram-estimator.py models/qwen3-235B-Q3_K-XL/UD-Q3_K_XL/Qwen3-235B-A22B-Instruct-2507-UD-Q3_K_XL-00001-of-00003.gguf --contexts 65536 131072 262144
|
||||
|
||||
--- Memory Footprint Estimation ---
|
||||
Context Size | Context Memory | Est. Total VRAM
|
||||
---------------------------------------------------
|
||||
65,536 | 11.75 GiB | 110.75 GiB
|
||||
131,072 | 23.50 GiB | 122.50 GiB
|
||||
262,144 | 47.00 GiB | 146.00 GiB
|
||||
```
|
||||
|
||||
* **Takeaway:**
|
||||
|
||||
* With 128GB, you can go up to \~130k context on this Qwen 235B quantized model.
|
||||
* If you go higher, you will OOM—even before context reaches the model's max.
|
||||
|
||||
---
|
||||
|
||||
## 4. Notes
|
||||
|
||||
* “Est. Total VRAM” is the minimum you’ll need for the model + context, but does not include OS, other processes, or toolbox/container overhead—leave a margin.
|
||||
* For detailed methodology or custom scenarios, check the script source.
|
||||
* Benchmark speed for large context sizes is often the real bottleneck—see `docs/benchmarks.md` for real throughput figures.
|
||||
|
||||
---
|
||||
|
||||
## 5. Related
|
||||
|
||||
* Main README section [Memory Planning & VRAM Estimator](../Readme#4--memory-planning--vram-estimator)
|
||||
* [docs/benchmarks.md](benchmarks.md) for full speed/compat charts
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
# List of all known toolboxes and their configurations
|
||||
declare -A TOOLBOXES
|
||||
|
||||
TOOLBOXES["llama-vulkan-amdvlk"]="docker.io/kyuz0/amd-r9700-toolboxes:vulkan-amdvlk --device /dev/dri --group-add video --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-vulkan-radv"]="docker.io/kyuz0/amd-r9700-toolboxes:vulkan-radv --device /dev/dri --group-add video --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-6.4.4"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-6.4.4 --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-6.4.4-rocwmma"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-6.4.4-rocwmma --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7.1"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.1 --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7.1-rocwmma"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.1-rocwmma --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7.9"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.9 --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7.9-rocwmma"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7.9-rocwmma --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7-nightly"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7-nightly --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
TOOLBOXES["llama-rocm-7-nightly-rocwmma"]="docker.io/kyuz0/amd-r9700-toolboxes:rocm-7-nightly-rocwmma --device /dev/dri --device /dev/kfd --group-add video --group-add render --group-add sudo --security-opt seccomp=unconfined"
|
||||
|
||||
function usage() {
|
||||
echo "Usage: $0 [all|toolbox-name1 toolbox-name2 ...]"
|
||||
echo "Available toolboxes:"
|
||||
for name in "${!TOOLBOXES[@]}"; do
|
||||
echo " - $name"
|
||||
done
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check dependencies
|
||||
for cmd in podman toolbox; do
|
||||
command -v "$cmd" > /dev/null || { echo "Error: '$cmd' is not installed." >&2; exit 1; }
|
||||
done
|
||||
|
||||
if [ "$#" -lt 1 ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
# Determine which toolboxes to refresh
|
||||
if [ "$1" = "all" ]; then
|
||||
SELECTED_TOOLBOXES=("${!TOOLBOXES[@]}")
|
||||
else
|
||||
SELECTED_TOOLBOXES=()
|
||||
for arg in "$@"; do
|
||||
if [[ -v TOOLBOXES["$arg"] ]]; then
|
||||
SELECTED_TOOLBOXES+=("$arg")
|
||||
else
|
||||
echo "Error: Unknown toolbox '$arg'"
|
||||
usage
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Loop through selected toolboxes
|
||||
for name in "${SELECTED_TOOLBOXES[@]}"; do
|
||||
config="${TOOLBOXES[$name]}"
|
||||
image=$(echo "$config" | awk '{print $1}')
|
||||
options="${config#* }"
|
||||
|
||||
echo "🔄 Refreshing $name (image: $image)"
|
||||
|
||||
# Remove the toolbox if it exists
|
||||
if toolbox list | grep -q "$name"; then
|
||||
echo "🧹 Removing existing toolbox: $name"
|
||||
toolbox rm -f "$name"
|
||||
fi
|
||||
|
||||
echo "⬇️ Pulling latest image: $image"
|
||||
podman pull "$image"
|
||||
|
||||
# Identify current image ID/digest for this tag
|
||||
new_id="$(podman image inspect --format '{{.Id}}' "$image" 2>/dev/null || true)"
|
||||
new_digest="$(podman image inspect --format '{{.Digest}}' "$image" 2>/dev/null || true)"
|
||||
|
||||
echo "📦 Recreating toolbox: $name"
|
||||
toolbox create "$name" --image "$image" -- $options
|
||||
|
||||
# --- Cleanup: keep only the most recent image for this tag ---
|
||||
repo="${image%:*}"
|
||||
tag="${image##*:}"
|
||||
|
||||
# Remove any other local images still carrying this exact tag but not the newest digest
|
||||
while read -r id ref dig; do
|
||||
[[ "$id" != "$new_id" ]] && podman image rm -f "$id" >/dev/null 2>&1 || true
|
||||
done < <(podman images --digests --format '{{.ID}} {{.Repository}}:{{.Tag}} {{.Digest}}' \
|
||||
| awk -v ref="$image" -v ndig="$new_digest" '$2==ref && $3!=ndig')
|
||||
|
||||
# Remove dangling images from this repository (typically prior pulls of this tag)
|
||||
while read -r id; do
|
||||
podman image rm -f "$id" >/dev/null 2>&1 || true
|
||||
done < <(podman images --format '{{.ID}} {{.Repository}}:{{.Tag}}' \
|
||||
| awk -v r="$repo" '$2==r":<none>" {print $1}')
|
||||
# --- end cleanup ---
|
||||
|
||||
echo "✅ $name refreshed"
|
||||
echo
|
||||
done
|
||||
@@ -0,0 +1,111 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# rocm 6.4.3 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-6.4.4]
|
||||
name=ROCm6.4.4
|
||||
baseurl=https://repo.radeon.com/rocm/el9/6.4.4/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel \
|
||||
rocm-llvm rocm-device-libs hip-runtime-amd hip-devel \
|
||||
rocblas rocblas-devel hipblas hipblas-devel \
|
||||
rocminfo radeontop \
|
||||
git-core vim sudo rsync \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# rocm env
|
||||
ENV ROCM_PATH=/opt/rocm \
|
||||
HIP_PATH=/opt/rocm \
|
||||
HIP_CLANG_PATH=/opt/rocm/llvm/bin \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
# build
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive \
|
||||
&& cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
-DROCM_PATH=/opt/rocm \
|
||||
-DHIP_PATH=/opt/rocm \
|
||||
-DHIP_PLATFORM=amd \
|
||||
-DCMAKE_HIP_FLAGS="--rocm-path=/opt/rocm" \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# rocm 6.4.3 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-6.4.4]
|
||||
name=ROCm6.4.4
|
||||
baseurl=https://repo.radeon.com/rocm/el9/6.4.4/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc libgomp sudo \
|
||||
hip-runtime-amd rocblas hipblas \
|
||||
rocminfo radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# profile
|
||||
RUN printf '%s\n' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh && chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,127 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# rocm 6.4.4 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-6.4.4]
|
||||
name=ROCm6.4.4
|
||||
baseurl=https://repo.radeon.com/rocm/el9/6.4.4/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel ninja-build \
|
||||
rocm-llvm rocm-device-libs hip-runtime-amd hip-devel \
|
||||
rocblas rocblas-devel hipblas hipblas-devel rocm-cmake libomp-devel libomp \
|
||||
rocminfo radeontop \
|
||||
git-core vim sudo rsync \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# rocm env
|
||||
ENV ROCM_PATH=/opt/rocm \
|
||||
HIP_PATH=/opt/rocm \
|
||||
HIP_CLANG_PATH=/opt/rocm/llvm/bin \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH
|
||||
|
||||
# rocWMMA
|
||||
WORKDIR /opt
|
||||
COPY ./build-rocwmma.sh .
|
||||
RUN chmod +x build-rocwmma.sh && ./build-rocwmma.sh
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive
|
||||
|
||||
# overwrite upstream header with our local fixed version
|
||||
COPY ggml/src/ggml-cuda/hip_shfl_fix.h /opt/llama.cpp/ggml/src/ggml-cuda/hip_shfl_fix.h
|
||||
|
||||
# Apply # rocWMMA patch
|
||||
COPY ./apply-rocwmma-fix.sh /opt/apply-rocwmma-fix.sh
|
||||
RUN chmod +x /opt/apply-rocwmma-fix.sh && /opt/apply-rocwmma-fix.sh /opt/llama.cpp
|
||||
|
||||
# Build
|
||||
RUN set -euo pipefail \
|
||||
&& cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
-DGGML_HIP_ROCWMMA_FATTN=ON \
|
||||
-DROCM_PATH=/opt/rocm \
|
||||
-DHIP_PATH=/opt/rocm \
|
||||
-DHIP_PLATFORM=amd \
|
||||
-DCMAKE_HIP_FLAGS="--rocm-path=/opt/rocm -include /opt/llama.cpp/ggml/src/ggml-cuda/hip_shfl_fix.h -Wno-macro-redefined" \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# rocm 6.4.3 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-6.4.4]
|
||||
name=ROCm6.4.4
|
||||
baseurl=https://repo.radeon.com/rocm/el9/6.4.4/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc libgomp sudo \
|
||||
hip-runtime-amd rocblas hipblas \
|
||||
rocminfo radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# profile
|
||||
RUN printf '%s\n' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh && chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,125 @@
|
||||
# build
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel \
|
||||
radeontop git vim patch curl ninja-build tar xz aria2c \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# find & fetch the latest Linux 7.x.x tarball (gfx1151)
|
||||
WORKDIR /tmp
|
||||
ARG ROCM_MAJOR_VER=7
|
||||
ARG GFX=gfx120X-all
|
||||
RUN set -euo pipefail; \
|
||||
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
|
||||
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
|
||||
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
|
||||
| tr '<' '\n' \
|
||||
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
|
||||
| sort -V | tail -n1)"; \
|
||||
echo "Latest tarball: ${KEY}"; \
|
||||
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz
|
||||
RUN mkdir -p /opt/rocm-7.0 \
|
||||
&& tar xzf therock.tar.gz -C /opt/rocm-7.0 --strip-components=1
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git . \
|
||||
&& git clean -xdf \
|
||||
&& git submodule update --recursive
|
||||
|
||||
RUN cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# keep bin; drop headers/docs/static libs (retain llama.cpp for rpc binaries)
|
||||
RUN find /opt/rocm-7.0 -type f -name '*.a' -delete \
|
||||
&& rm -rf /opt/rocm-7.0/include /opt/rocm-7.0/share \
|
||||
/opt/rocm-7.0/llvm/include /opt/rocm-7.0/llvm/share
|
||||
|
||||
# runtime
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc radeontop vim \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
COPY --from=builder /opt/rocm-7.0 /opt/rocm-7.0
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# make /usr/local libs visible without touching env
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,126 @@
|
||||
# build
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel \
|
||||
radeontop git vim patch curl ninja-build tar xz aria2c \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# find & fetch the latest Linux 7.x.x tarball (gfx1151)
|
||||
WORKDIR /tmp
|
||||
ARG ROCM_MAJOR_VER=7
|
||||
ARG GFX=gfx120X-all
|
||||
RUN set -euo pipefail; \
|
||||
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
|
||||
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
|
||||
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
|
||||
| tr '<' '\n' \
|
||||
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\..*\.tar\.gz" \
|
||||
| sort -V | tail -n1)"; \
|
||||
echo "Latest tarball: ${KEY}"; \
|
||||
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz
|
||||
RUN mkdir -p /opt/rocm-7.0 \
|
||||
&& tar xzf therock.tar.gz -C /opt/rocm-7.0 --strip-components=1
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
WORKDIR /opt
|
||||
COPY ./build-rocwmma.sh .
|
||||
RUN chmod +x build-rocwmma.sh && ./build-rocwmma.sh
|
||||
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git . \
|
||||
&& git clean -xdf \
|
||||
&& git submodule update --recursive
|
||||
COPY ./apply-rocwmma-fix.sh /opt/apply-rocwmma-fix.sh
|
||||
RUN chmod +x /opt/apply-rocwmma-fix.sh && /opt/apply-rocwmma-fix.sh /opt/llama.cpp
|
||||
|
||||
RUN cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DGGML_HIP_ROCWMMA_FATTN=ON \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# runtime
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc radeontop vim \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
COPY --from=builder /opt/rocm-7.0 /opt/rocm-7.0
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# make /usr/local libs visible without touching env
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,111 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# rocm 7.1 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-7.1]
|
||||
name=ROCm7.1
|
||||
baseurl=https://repo.radeon.com/rocm/el9/7.1/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel ninja-build \
|
||||
rocm-llvm rocm-device-libs hip-runtime-amd hip-devel \
|
||||
rocblas rocblas-devel hipblas hipblas-devel rocm-cmake libomp-devel libomp \
|
||||
rocminfo radeontop \
|
||||
git-core vim sudo rsync \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# rocm env
|
||||
ENV ROCM_PATH=/opt/rocm \
|
||||
HIP_PATH=/opt/rocm \
|
||||
HIP_CLANG_PATH=/opt/rocm/llvm/bin \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
# build
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive \
|
||||
&& cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
-DGGML_CUDA_ENABLE_UNIFIED_MEMORY=ON \
|
||||
-DROCM_PATH=/opt/rocm \
|
||||
-DHIP_PATH=/opt/rocm \
|
||||
-DHIP_PLATFORM=amd \
|
||||
-DCMAKE_HIP_FLAGS="--rocm-path=/opt/rocm" \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# rocm 7.1 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-7.1]
|
||||
name=ROCm7.1
|
||||
baseurl=https://repo.radeon.com/rocm/el9/7.1/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc libgomp sudo \
|
||||
hip-runtime-amd rocblas hipblas \
|
||||
rocminfo radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# profile
|
||||
RUN printf '%s\n' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh && chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,122 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# rocm 7.1 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-7.1]
|
||||
name=ROCm7.1
|
||||
baseurl=https://repo.radeon.com/rocm/el9/7.1/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel ninja-build \
|
||||
rocm-llvm rocm-device-libs hip-runtime-amd hip-devel \
|
||||
rocblas rocblas-devel hipblas hipblas-devel rocm-cmake libomp-devel libomp \
|
||||
rocminfo radeontop \
|
||||
git-core vim sudo rsync \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# rocm env
|
||||
ENV ROCM_PATH=/opt/rocm \
|
||||
HIP_PATH=/opt/rocm \
|
||||
HIP_CLANG_PATH=/opt/rocm/llvm/bin \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:$PATH
|
||||
|
||||
# rocWMMA
|
||||
WORKDIR /opt
|
||||
COPY ./build-rocwmma.sh .
|
||||
RUN chmod +x build-rocwmma.sh && ./build-rocwmma.sh
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
# Apply # rocWMMA patch
|
||||
COPY ./apply-rocwmma-fix.sh /opt/apply-rocwmma-fix.sh
|
||||
RUN chmod +x /opt/apply-rocwmma-fix.sh && /opt/apply-rocwmma-fix.sh /opt/llama.cpp
|
||||
|
||||
# build
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive \
|
||||
&& cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
-DGGML_CUDA_ENABLE_UNIFIED_MEMORY=ON \
|
||||
-DGGML_HIP_ROCWMMA_FATTN=ON \
|
||||
-DROCM_PATH=/opt/rocm \
|
||||
-DHIP_PATH=/opt/rocm \
|
||||
-DHIP_PLATFORM=amd \
|
||||
-DCMAKE_HIP_FLAGS="--rocm-path=/opt/rocm" \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# rocm 7.1 repo
|
||||
RUN <<'EOF'
|
||||
tee /etc/yum.repos.d/rocm.repo <<REPO
|
||||
[ROCm-7.1]
|
||||
name=ROCm7.1
|
||||
baseurl=https://repo.radeon.com/rocm/el9/7.1/main
|
||||
enabled=1
|
||||
priority=50
|
||||
gpgcheck=1
|
||||
gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
|
||||
REPO
|
||||
EOF
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 \
|
||||
--exclude='*sdk*' --exclude='*samples*' --exclude='*-doc*' --exclude='*-docs*' \
|
||||
install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc libgomp sudo \
|
||||
hip-runtime-amd rocblas hipblas \
|
||||
rocminfo radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# profile
|
||||
RUN printf '%s\n' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh && chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,124 @@
|
||||
# build
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel \
|
||||
radeontop git vim patch curl ninja-build tar xz aria2c \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# find & fetch the latest Linux 7.x.x rc tarball (gfx1151)
|
||||
WORKDIR /tmp
|
||||
ARG ROCM_MAJOR_VER=7
|
||||
ARG GFX=gfx120X-all
|
||||
RUN set -euo pipefail; \
|
||||
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
|
||||
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
|
||||
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
|
||||
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\.[0-9]\+\.[0-9]\+rc[0-9]\{8\}\.tar\.gz" \
|
||||
| sort | tail -n1)"; \
|
||||
echo "Latest tarball: ${KEY}"; \
|
||||
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz
|
||||
RUN mkdir -p /opt/rocm-7.0 \
|
||||
&& tar xzf therock.tar.gz -C /opt/rocm-7.0 --strip-components=1
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git . \
|
||||
&& git clean -xdf \
|
||||
&& git submodule update --recursive
|
||||
|
||||
RUN cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DLLAMA_HIP_UMA=ON \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# keep bin; drop headers/docs/static libs (retain llama.cpp for rpc binaries)
|
||||
RUN find /opt/rocm-7.0 -type f -name '*.a' -delete \
|
||||
&& rm -rf /opt/rocm-7.0/include /opt/rocm-7.0/share \
|
||||
/opt/rocm-7.0/llvm/include /opt/rocm-7.0/llvm/share
|
||||
|
||||
# runtime
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc radeontop vim \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
COPY --from=builder /opt/rocm-7.0 /opt/rocm-7.0
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# make /usr/local libs visible without touching env
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,126 @@
|
||||
# build
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
make gcc cmake lld clang clang-devel compiler-rt libcurl-devel \
|
||||
radeontop git vim patch curl ninja-build tar xz aria2c \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# find & fetch the latest Linux 7.x.x rc tarball (gfx1151)
|
||||
WORKDIR /tmp
|
||||
ARG ROCM_MAJOR_VER=7
|
||||
ARG GFX=gfx120X-all
|
||||
RUN set -euo pipefail; \
|
||||
BASE="https://therock-nightly-tarball.s3.amazonaws.com"; \
|
||||
PREFIX="therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}"; \
|
||||
KEY="$(curl -s "${BASE}?list-type=2&prefix=${PREFIX}" \
|
||||
| grep -o "therock-dist-linux-${GFX}-${ROCM_MAJOR_VER}\.[0-9]\+\.[0-9]\+rc[0-9]\{8\}\.tar\.gz" \
|
||||
| sort | tail -n1)"; \
|
||||
echo "Latest tarball: ${KEY}"; \
|
||||
aria2c -x 16 -s 16 -j 16 --file-allocation=none "${BASE}/${KEY}" -o therock.tar.gz
|
||||
RUN mkdir -p /opt/rocm-7.0 \
|
||||
&& tar xzf therock.tar.gz -C /opt/rocm-7.0 --strip-components=1
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
WORKDIR /opt
|
||||
COPY ./build-rocwmma.sh .
|
||||
RUN chmod +x build-rocwmma.sh && ./build-rocwmma.sh
|
||||
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git . \
|
||||
&& git clean -xdf \
|
||||
&& git submodule update --recursive
|
||||
|
||||
COPY ./apply-rocwmma-fix.sh /opt/apply-rocwmma-fix.sh
|
||||
RUN chmod +x /opt/apply-rocwmma-fix.sh && /opt/apply-rocwmma-fix.sh /opt/llama.cpp
|
||||
|
||||
RUN cmake -S . -B build \
|
||||
-DGGML_HIP=ON \
|
||||
-DAMDGPU_TARGETS=gfx1201 \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DGGML_HIP_ROCWMMA_FATTN=ON \
|
||||
&& cmake --build build --config Release -- -j$(nproc) \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# runtime
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc radeontop vim \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
COPY --from=builder /opt/rocm-7.0 /opt/rocm-7.0
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
ENV ROCM_PATH=/opt/rocm-7.0 \
|
||||
HIP_PLATFORM=amd \
|
||||
HIP_PATH=/opt/rocm-7.0 \
|
||||
HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin \
|
||||
HIP_INCLUDE_PATH=/opt/rocm-7.0/include \
|
||||
HIP_LIB_PATH=/opt/rocm-7.0/lib \
|
||||
HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode \
|
||||
PATH=/opt/rocm-7.0/bin:/opt/rocm-7.0/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
LD_LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64:/opt/rocm-7.0/llvm/lib \
|
||||
LIBRARY_PATH=/opt/rocm-7.0/lib:/opt/rocm-7.0/lib64 \
|
||||
CPATH=/opt/rocm-7.0/include \
|
||||
PKG_CONFIG_PATH=/opt/rocm-7.0/lib/pkgconfig
|
||||
|
||||
RUN printf '%s\n' \
|
||||
'export ROCM_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_PLATFORM=amd' \
|
||||
'export HIP_PATH=/opt/rocm-7.0' \
|
||||
'export HIP_CLANG_PATH=/opt/rocm-7.0/llvm/bin' \
|
||||
'export HIP_INCLUDE_PATH=/opt/rocm-7.0/include' \
|
||||
'export HIP_LIB_PATH=/opt/rocm-7.0/lib' \
|
||||
'export HIP_DEVICE_LIB_PATH=/opt/rocm-7.0/lib/llvm/amdgcn/bitcode' \
|
||||
'export PATH="$ROCM_PATH/bin:$HIP_CLANG_PATH:$PATH"' \
|
||||
'export LD_LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64:$ROCM_PATH/llvm/lib"' \
|
||||
'export LIBRARY_PATH="$HIP_LIB_PATH:$ROCM_PATH/lib:$ROCM_PATH/lib64"' \
|
||||
'export CPATH="$HIP_INCLUDE_PATH"' \
|
||||
'export PKG_CONFIG_PATH="$ROCM_PATH/lib/pkgconfig"' \
|
||||
'export ROCBLAS_USE_HIPBLASLT=1' \
|
||||
> /etc/profile.d/rocm.sh \
|
||||
&& chmod +x /etc/profile.d/rocm.sh \
|
||||
&& echo 'source /etc/profile.d/rocm.sh' >> /etc/bashrc
|
||||
|
||||
# make /usr/local libs visible without touching env
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,77 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
git vim \
|
||||
make gcc cmake ninja-build lld clang clang-devel compiler-rt libcurl-devel \
|
||||
vulkan-loader-devel vulkaninfo mesa-vulkan-drivers \
|
||||
radeontop glslc wget \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# amdvlk
|
||||
RUN curl -L -o /tmp/amdvlk-2025.Q2.1.x86_64.rpm \
|
||||
https://github.com/GPUOpen-Drivers/AMDVLK/releases/download/v-2025.Q2.1/amdvlk-2025.Q2.1.x86_64.rpm \
|
||||
&& dnf -y install /tmp/amdvlk-*.rpm \
|
||||
&& rm -f /tmp/amdvlk-*.rpm
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
# build
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive \
|
||||
&& cmake -S . -B build -G Ninja \
|
||||
-DGGML_VULKAN=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=ON \
|
||||
-DLLAMA_BUILD_SERVER=ON \
|
||||
&& cmake --build build --config Release \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc \
|
||||
vulkan-loader vulkan-loader-devel vulkaninfo mesa-vulkan-drivers radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# amdvlk
|
||||
RUN curl -L -o /tmp/amdvlk-2025.Q2.1.x86_64.rpm \
|
||||
https://github.com/GPUOpen-Drivers/AMDVLK/releases/download/v-2025.Q2.1/amdvlk-2025.Q2.1.x86_64.rpm \
|
||||
&& microdnf -y install /tmp/amdvlk-*.rpm \
|
||||
&& rm -f /tmp/amdvlk-*.rpm
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/ /usr/
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,65 @@
|
||||
# build stage
|
||||
FROM registry.fedoraproject.org/fedora:43 AS builder
|
||||
|
||||
# deps
|
||||
RUN dnf -y --nodocs --setopt=install_weak_deps=False install \
|
||||
git vim \
|
||||
make gcc cmake ninja-build lld clang clang-devel compiler-rt libcurl-devel \
|
||||
vulkan-loader-devel vulkaninfo mesa-vulkan-drivers \
|
||||
radeontop glslc \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# llama.cpp
|
||||
WORKDIR /opt/llama.cpp
|
||||
RUN git clone --recursive https://github.com/ggerganov/llama.cpp.git .
|
||||
|
||||
# build
|
||||
RUN git clean -xdf \
|
||||
&& git submodule update --recursive \
|
||||
&& cmake -S . -B build -G Ninja \
|
||||
-DGGML_VULKAN=ON \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_RPC=ON \
|
||||
-DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DLLAMA_BUILD_TESTS=OFF \
|
||||
-DLLAMA_BUILD_EXAMPLES=ON \
|
||||
-DLLAMA_BUILD_SERVER=ON \
|
||||
&& cmake --build build --config Release \
|
||||
&& cmake --install build --config Release
|
||||
|
||||
# libs
|
||||
RUN find /opt/llama.cpp/build -type f -name 'lib*.so*' -exec cp {} /usr/lib64/ \; \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
|
||||
# runtime stage
|
||||
FROM registry.fedoraproject.org/fedora-minimal:43
|
||||
|
||||
# runtime deps
|
||||
RUN microdnf -y --nodocs --setopt=install_weak_deps=0 install \
|
||||
bash ca-certificates libatomic libstdc++ libgcc \
|
||||
vulkan-loader vulkan-loader-devel vulkaninfo mesa-vulkan-drivers radeontop \
|
||||
&& microdnf clean all && rm -rf /var/cache/dnf/*
|
||||
|
||||
# copy
|
||||
COPY --from=builder /usr/ /usr/
|
||||
COPY --from=builder /usr/local/ /usr/local/
|
||||
COPY --from=builder /opt/llama.cpp/build/bin/rpc-* /usr/local/bin/
|
||||
|
||||
# ld
|
||||
RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf \
|
||||
&& echo "/usr/local/lib64" >> /etc/ld.so.conf.d/local.conf \
|
||||
&& ldconfig \
|
||||
&& cp -n /usr/local/lib/libllama*.so* /usr/lib64/ 2>/dev/null || true \
|
||||
&& ldconfig
|
||||
|
||||
# helper
|
||||
COPY gguf-vram-estimator.py /usr/local/bin/gguf-vram-estimator.py
|
||||
RUN chmod +x /usr/local/bin/gguf-vram-estimator.py
|
||||
|
||||
# shell
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/bin/bash
|
||||
|
||||
# apply-rocwmma-fix.sh - Apply rocWMMA compatibility fixes to llama.cpp
|
||||
# Usage: ./apply-rocwmma-fix.sh <path-to-llama.cpp-directory>
|
||||
# Source: https://github.com/lhl/strix-halo-testing/blob/main/llm-bench/apply-rocwmma-fix.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LLAMA_DIR="${1:-}"
|
||||
|
||||
if [[ -z "$LLAMA_DIR" ]]; then
|
||||
echo "Usage: $0 <path-to-llama.cpp-directory>"
|
||||
echo ""
|
||||
echo "This script applies rocWMMA compatibility fixes to a llama.cpp checkout."
|
||||
echo "The fixes resolve warp synchronization mask type conflicts between"
|
||||
echo "ROCm headers and CUDA-style code when building with GGML_HIP_ROCWMMA_FATTN=ON."
|
||||
echo ""
|
||||
echo "Example:"
|
||||
echo " $0 ./llama.cpp"
|
||||
echo " $0 /path/to/your/llama.cpp"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -d "$LLAMA_DIR" ]]; then
|
||||
echo "Error: Directory '$LLAMA_DIR' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$LLAMA_DIR/CMakeLists.txt" ]] || ! grep -q "llama" "$LLAMA_DIR/CMakeLists.txt" 2>/dev/null; then
|
||||
echo "Error: '$LLAMA_DIR' does not appear to be a llama.cpp directory"
|
||||
echo "Expected to find CMakeLists.txt with 'llama' references"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VENDOR_HIP_FILE="$LLAMA_DIR/ggml/src/ggml-cuda/vendors/hip.h"
|
||||
|
||||
if [[ ! -f "$VENDOR_HIP_FILE" ]]; then
|
||||
echo "Error: HIP vendor header not found at: $VENDOR_HIP_FILE"
|
||||
echo "This script requires a llama.cpp version with HIP support"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Applying rocWMMA compatibility fixes to: $LLAMA_DIR"
|
||||
echo ""
|
||||
|
||||
# Check if fixes are already applied
|
||||
if grep -q "GGML_HIP_WARP_MASK" "$VENDOR_HIP_FILE" 2>/dev/null; then
|
||||
echo "rocWMMA fixes appear to already be applied (found GGML_HIP_WARP_MASK)"
|
||||
echo "To reapply, please first revert changes and run this script again"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Step 1: Modifying HIP vendor header..."
|
||||
|
||||
# Backup the original file
|
||||
cp "$VENDOR_HIP_FILE" "$VENDOR_HIP_FILE.backup"
|
||||
|
||||
# Find the line with __shfl_sync and __shfl_xor_sync definitions
|
||||
SHFL_LINE=$(grep -n "^#define __shfl_sync" "$VENDOR_HIP_FILE" | head -1 | cut -d: -f1)
|
||||
|
||||
if [[ -z "$SHFL_LINE" ]]; then
|
||||
echo "Error: Could not find __shfl_sync macro definition in $VENDOR_HIP_FILE"
|
||||
echo "This script may need updates for this version of llama.cpp"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create a temporary file with the fix
|
||||
{
|
||||
# Print lines before the __shfl_sync definition
|
||||
head -n $((SHFL_LINE - 1)) "$VENDOR_HIP_FILE"
|
||||
|
||||
# Add our conditional compilation block
|
||||
cat << 'EOF'
|
||||
#ifdef GGML_HIP_ROCWMMA_FATTN
|
||||
// ROCm requires 64-bit masks for __shfl_*_sync functions
|
||||
#define GGML_HIP_WARP_MASK 0xFFFFFFFFFFFFFFFFULL
|
||||
#else
|
||||
#define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width)
|
||||
#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
|
||||
#define GGML_HIP_WARP_MASK 0xFFFFFFFF
|
||||
#endif
|
||||
EOF
|
||||
|
||||
# Skip the original __shfl_sync and __shfl_xor_sync lines and print the rest
|
||||
tail -n +$((SHFL_LINE + 2)) "$VENDOR_HIP_FILE"
|
||||
|
||||
} > "$VENDOR_HIP_FILE.tmp"
|
||||
|
||||
mv "$VENDOR_HIP_FILE.tmp" "$VENDOR_HIP_FILE"
|
||||
|
||||
echo " ✓ Added conditional GGML_HIP_WARP_MASK macro to vendor header"
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Replacing hardcoded warp masks in CUDA files..."
|
||||
|
||||
# Find all .cu and .cuh files in the ggml/src/ggml-cuda directory
|
||||
CUDA_FILES=($(find "$LLAMA_DIR/ggml/src/ggml-cuda" -name "*.cu" -o -name "*.cuh" 2>/dev/null | sort))
|
||||
|
||||
if [[ ${#CUDA_FILES[@]} -eq 0 ]]; then
|
||||
echo "Warning: No CUDA files found in $LLAMA_DIR/ggml/src/ggml-cuda"
|
||||
echo "This may be expected for some llama.cpp versions"
|
||||
else
|
||||
MODIFIED_COUNT=0
|
||||
|
||||
for file in "${CUDA_FILES[@]}"; do
|
||||
# Check if file contains the hardcoded masks
|
||||
if grep -q "0xFFFFFFFF\|0xffffffff" "$file" 2>/dev/null; then
|
||||
# Create backup
|
||||
cp "$file" "$file.backup"
|
||||
|
||||
# Replace both uppercase and lowercase versions
|
||||
sed -i 's/0xFFFFFFFF/GGML_HIP_WARP_MASK/g; s/0xffffffff/GGML_HIP_WARP_MASK/g' "$file"
|
||||
|
||||
MODIFIED_COUNT=$((MODIFIED_COUNT + 1))
|
||||
echo " ✓ Modified: $(basename "$file")"
|
||||
fi
|
||||
done
|
||||
|
||||
echo " ✓ Modified $MODIFIED_COUNT CUDA files"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 3: Verification..."
|
||||
|
||||
# Verify the vendor header was modified correctly
|
||||
if grep -q "GGML_HIP_ROCWMMA_FATTN" "$VENDOR_HIP_FILE" && grep -q "GGML_HIP_WARP_MASK" "$VENDOR_HIP_FILE"; then
|
||||
echo " ✓ Vendor header modification verified"
|
||||
else
|
||||
echo " ✗ Vendor header modification failed"
|
||||
# Restore backup
|
||||
mv "$VENDOR_HIP_FILE.backup" "$VENDOR_HIP_FILE"
|
||||
echo " ✓ Restored original vendor header"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🎉 rocWMMA compatibility fixes applied successfully!"
|
||||
echo ""
|
||||
echo "What was changed:"
|
||||
echo " • Added conditional GGML_HIP_WARP_MASK macro to ggml/src/ggml-cuda/vendors/hip.h"
|
||||
echo " • Replaced hardcoded 0xFFFFFFFF/0xffffffff with GGML_HIP_WARP_MASK in CUDA files"
|
||||
echo ""
|
||||
echo "Behavior:"
|
||||
echo " • For regular HIP builds: GGML_HIP_WARP_MASK = 0xFFFFFFFF (no change)"
|
||||
echo " • For rocWMMA builds: GGML_HIP_WARP_MASK = 0xFFFFFFFFFFFFFFFFULL (64-bit masks)"
|
||||
echo ""
|
||||
echo "To build with rocWMMA support, use:"
|
||||
echo " cmake -B build -S '$LLAMA_DIR' -DGGML_HIP=ON -DAMDGPU_TARGETS=\"gfx1201\" -DGGML_HIP_ROCWMMA_FATTN=ON"
|
||||
echo ""
|
||||
echo "Backup files were created with .backup extension in case you need to revert."
|
||||
echo ""
|
||||
echo "Done! Your llama.cpp checkout now supports rocWMMA builds."
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
# Source: https://github.com/lhl/strix-halo-testing/blob/main/llm-bench/build-rocwmma.sh
|
||||
git clone https://github.com/ROCm/rocWMMA
|
||||
cd rocWMMA
|
||||
|
||||
# --- BEGIN: make OpenMP explicit for ROCm toolchains (drop-in) ---
|
||||
# find libomp (check ROCM_PATH first, then system)
|
||||
CANDIDATES=(
|
||||
"${ROCM_PATH}/llvm/lib/libomp.so"
|
||||
"${ROCM_PATH}/llvm/lib/libomp.a"
|
||||
"/usr/lib64/libomp.so"
|
||||
"/usr/lib64/libomp.a"
|
||||
"/usr/local/lib/libomp.so"
|
||||
)
|
||||
FOUND_LIBOMP=""
|
||||
for p in "${CANDIDATES[@]}"; do
|
||||
if [ -f "$p" ]; then
|
||||
FOUND_LIBOMP="$p"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
CMAKE_OPTS=""
|
||||
if [ -n "$FOUND_LIBOMP" ]; then
|
||||
# directory & basename
|
||||
OMP_LIB_DIR="$(dirname "$FOUND_LIBOMP")"
|
||||
OMP_LIB_BASENAME="$(basename "$FOUND_LIBOMP")"
|
||||
# set cache vars so FindOpenMP will succeed
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_CXX_FLAGS=-fopenmp=libomp"
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_C_FLAGS=-fopenmp=libomp"
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_CXX_LIB_NAMES=omp"
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_C_LIB_NAMES=omp"
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_LIBRARY=${FOUND_LIBOMP}"
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_INCLUDE_DIR=${ROCM_PATH}/llvm/include"
|
||||
export LD_LIBRARY_PATH="${OMP_LIB_DIR}${LD_LIBRARY_PATH:+:}$LD_LIBRARY_PATH"
|
||||
export CXXFLAGS="-fopenmp=libomp ${CXXFLAGS:-}"
|
||||
export LDFLAGS="-L${OMP_LIB_DIR} -lomp ${LDFLAGS:-}"
|
||||
else
|
||||
# fallback: force flags so FindOpenMP might at least get flags
|
||||
CMAKE_OPTS="${CMAKE_OPTS} -DOpenMP_CXX_FLAGS=-fopenmp=libomp -DOpenMP_C_FLAGS=-fopenmp=libomp"
|
||||
export CXXFLAGS="-fopenmp=libomp ${CXXFLAGS:-}"
|
||||
export LDFLAGS="${LDFLAGS:-} -lomp"
|
||||
fi
|
||||
# --- END: make OpenMP explicit ---
|
||||
|
||||
CC=$ROCM_PATH/llvm/bin/amdclang \
|
||||
CXX=$ROCM_PATH/llvm/bin/amdclang++ \
|
||||
cmake -B build -S . -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX=$ROCM_PATH \
|
||||
-DROCWMMA_BUILD_TESTS=OFF \
|
||||
-DROCWMMA_BUILD_SAMPLES=OFF \
|
||||
-DGPU_TARGETS="gfx1201" \
|
||||
-DOpenMP_CXX_FLAGS="-fopenmp=libomp" \
|
||||
-DOpenMP_C_FLAGS="-fopenmp=libomp" \
|
||||
-DOpenMP_omp_LIBRARY="/usr/lib64/libomp.so" \
|
||||
-DOpenMP_CXX_LIB_NAMES="omp" \
|
||||
-DOpenMP_C_LIB_NAMES="omp" \
|
||||
-DOpenMP_INCLUDE_DIRS="/usr/lib64/clang/19/include"
|
||||
|
||||
cmake --install build
|
||||
sudo cmake --install build
|
||||
@@ -0,0 +1,14 @@
|
||||
#ifndef HIP_SHFL_FIX_H
|
||||
#define HIP_SHFL_FIX_H
|
||||
#ifdef __HIP_PLATFORM_AMD__
|
||||
#ifndef __shfl_sync
|
||||
#define __shfl_sync(mask,var,srcLane,width) __shfl((var),(srcLane),(width))
|
||||
#endif
|
||||
#ifndef __shfl_up_sync
|
||||
#define __shfl_up_sync(mask,var,delta,width) __shfl_up((var),(delta),(width))
|
||||
#endif
|
||||
#ifndef __shfl_xor_sync
|
||||
#define __shfl_xor_sync(mask,var,laneMask,width) __shfl_xor((var),(laneMask),(width))
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import argparse
|
||||
import math
|
||||
from typing import Dict, Any, List
|
||||
|
||||
# GGUF constants
|
||||
GGUF_MAGIC = 0x46554747
|
||||
GGUF_VALUE_TYPE = {
|
||||
0: "UINT8", 1: "INT8", 2: "UINT16", 3: "INT16", 4: "UINT32",
|
||||
5: "INT32", 6: "FLOAT32", 7: "BOOL", 8: "STRING", 9: "ARRAY",
|
||||
}
|
||||
|
||||
class GGUFMetadataReader:
|
||||
"""A minimal reader to get only the necessary KV metadata for cache calculation."""
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self.metadata: Dict[str, Any] = {}
|
||||
|
||||
def read(self):
|
||||
with open(self.path, "rb") as f:
|
||||
self.f = f
|
||||
magic, _, _, metadata_kv_count = struct.unpack("<IIQQ", self.f.read(24))
|
||||
if magic != GGUF_MAGIC: raise ValueError("Invalid GGUF magic number")
|
||||
self._read_metadata(metadata_kv_count)
|
||||
return self
|
||||
|
||||
def _read_string(self) -> str:
|
||||
(length,) = struct.unpack("<Q", self.f.read(8))
|
||||
return self.f.read(length).decode("utf-8", errors="replace")
|
||||
|
||||
def _read_value(self, value_type_idx: int):
|
||||
value_type = GGUF_VALUE_TYPE.get(value_type_idx)
|
||||
if not value_type: raise ValueError(f"Unknown GGUF value type: {value_type_idx}")
|
||||
if value_type == "STRING": return self._read_string()
|
||||
if value_type == "UINT32": return struct.unpack("<I", self.f.read(4))[0]
|
||||
if value_type == "INT32": return struct.unpack("<i", self.f.read(4))[0]
|
||||
self._skip_value(value_type_idx)
|
||||
|
||||
def _skip_value(self, value_type_idx: int):
|
||||
value_type = GGUF_VALUE_TYPE.get(value_type_idx)
|
||||
if not value_type: return
|
||||
if value_type in ("UINT8", "INT8", "BOOL"): self.f.seek(1, 1)
|
||||
elif value_type in ("UINT16", "INT16"): self.f.seek(2, 1)
|
||||
elif value_type in ("UINT32", "INT32", "FLOAT32"): self.f.seek(4, 1)
|
||||
elif value_type == "STRING":
|
||||
(length,) = struct.unpack("<Q", self.f.read(8))
|
||||
self.f.seek(length, 1)
|
||||
elif value_type == "ARRAY":
|
||||
(array_type_idx, count) = struct.unpack("<IQ", self.f.read(12))
|
||||
type_map = {0:1, 1:1, 2:2, 3:2, 4:4, 5:4, 6:4, 7:1, 10:8, 11:8, 12:8}
|
||||
element_size = type_map.get(array_type_idx)
|
||||
if element_size: self.f.seek(count * element_size, 1)
|
||||
else:
|
||||
for _ in range(count): self._skip_value(8)
|
||||
|
||||
def _read_metadata(self, count: int):
|
||||
keys_to_read = {"general.architecture", "general.name"}
|
||||
arch_specific_keys_added = False
|
||||
for _ in range(count):
|
||||
key = self._read_string()
|
||||
(value_type_idx,) = struct.unpack("<I", self.f.read(4))
|
||||
if not arch_specific_keys_added and "general.architecture" in self.metadata:
|
||||
prefix = self.metadata["general.architecture"]
|
||||
keys_to_read.update({
|
||||
f"{prefix}.block_count", f"{prefix}.context_length",
|
||||
f"{prefix}.attention.head_count_kv", f"{prefix}.attention.key_length",
|
||||
f"{prefix}.attention.value_length", f"{prefix}.attention.sliding_window_size"
|
||||
})
|
||||
arch_specific_keys_added = True
|
||||
if key in keys_to_read:
|
||||
self.metadata[key] = self._read_value(value_type_idx)
|
||||
else:
|
||||
self._skip_value(value_type_idx)
|
||||
|
||||
def get_total_model_size_from_disk(gguf_file_path: str) -> int:
|
||||
"""Calculates the total model size by finding all parts on disk."""
|
||||
match = re.search(r'-(\d{5})-of-(\d{5})\.gguf$', gguf_file_path, re.IGNORECASE)
|
||||
if not match:
|
||||
return os.path.getsize(gguf_file_path)
|
||||
|
||||
base_path = gguf_file_path[:match.start()]
|
||||
total_parts_str = match.group(2)
|
||||
total_parts = int(total_parts_str)
|
||||
total_size, found_parts = 0, 0
|
||||
for i in range(1, total_parts + 1):
|
||||
part_file_name = f"{base_path}-{i:05d}-of-{total_parts_str}.gguf"
|
||||
if os.path.exists(part_file_name):
|
||||
total_size += os.path.getsize(part_file_name)
|
||||
found_parts += 1
|
||||
if found_parts != total_parts:
|
||||
print(f"WARNING: Expected {total_parts} parts, found {found_parts}. Size calculation may be incomplete.", file=sys.stderr)
|
||||
return total_size
|
||||
|
||||
def format_mem(size_bytes):
|
||||
mib = size_bytes / (1024 * 1024)
|
||||
if mib < 1024: return f"{mib:8.2f} MiB"
|
||||
return f"{mib / 1024:8.2f} GiB"
|
||||
|
||||
def run_estimator(gguf_file: str, context_sizes: List[int], overhead_gib: float):
|
||||
try:
|
||||
reader = GGUFMetadataReader(gguf_file).read()
|
||||
metadata = reader.metadata
|
||||
prefix = metadata.get("general.architecture")
|
||||
if not prefix: raise KeyError("Could not read 'general.architecture' from model metadata.")
|
||||
|
||||
model_size_bytes = get_total_model_size_from_disk(gguf_file)
|
||||
overhead_bytes = int(overhead_gib * 1024**3)
|
||||
|
||||
n_layers = metadata[f"{prefix}.block_count"]
|
||||
n_head_kv = metadata[f"{prefix}.attention.head_count_kv"]
|
||||
training_context = metadata.get(f"{prefix}.context_length", 0)
|
||||
n_embd_head_k = metadata[f"{prefix}.attention.key_length"]
|
||||
n_embd_head_v = metadata[f"{prefix}.attention.value_length"]
|
||||
swa_window_size = metadata.get(f"{prefix}.attention.sliding_window_size", 0)
|
||||
|
||||
is_scout_model = "scout" in metadata.get("general.name", "").lower()
|
||||
if is_scout_model and swa_window_size == 0: n_layers_swa, n_layers_full, swa_window_size = 36, 12, 8192
|
||||
elif swa_window_size > 0: n_layers_swa, n_layers_full = n_layers, 0
|
||||
else: n_layers_swa, n_layers_full = 0, n_layers
|
||||
|
||||
print(f"\n--- Model '{metadata.get('general.name', 'N/A')}' ---")
|
||||
if training_context > 0: print(f"Max Context: {training_context:,} tokens")
|
||||
print(f"Model Size: {format_mem(model_size_bytes).strip()} (from file size)")
|
||||
print(f"Incl. Overhead: {overhead_gib:.2f} GiB (for compute buffer, etc. adjustable via --overhead)")
|
||||
|
||||
if training_context > 0:
|
||||
context_sizes = sorted(list(set([c for c in context_sizes if c <= training_context] + [c for c in [training_context] if c not in context_sizes])))
|
||||
else: context_sizes = sorted(context_sizes)
|
||||
|
||||
bytes_per_token_per_layer = n_head_kv * (n_embd_head_k + n_embd_head_v) * 2
|
||||
|
||||
print("\n--- Memory Footprint Estimation ---")
|
||||
print(f"{'Context Size':>15s} | {'Context Memory':>15s} | {'Est. Total VRAM':>15s}")
|
||||
print("-" * 51)
|
||||
for n_ctx in context_sizes:
|
||||
mem_full = n_ctx * n_layers_full * bytes_per_token_per_layer
|
||||
mem_swa = min(n_ctx, swa_window_size) * n_layers_swa * bytes_per_token_per_layer
|
||||
kv_cache_bytes = mem_full + mem_swa
|
||||
total_bytes = model_size_bytes + kv_cache_bytes + overhead_bytes
|
||||
print(f"{n_ctx:>15,} | {format_mem(kv_cache_bytes):>15s} | {format_mem(total_bytes):>15s}")
|
||||
|
||||
except (FileNotFoundError, ValueError, struct.error, NotImplementedError, KeyError) as e:
|
||||
print(f"\nError: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Calculate VRAM requirements for a GGUF model, including a configurable overhead for compute buffers.",
|
||||
formatter_class=argparse.RawTextHelpFormatter
|
||||
)
|
||||
parser.add_argument("gguf_file", help="Path to the GGUF model file (any part of a multi-part model).")
|
||||
parser.add_argument("-c", "--contexts", nargs='+', type=int, default=[4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576], help="Space-separated list of context sizes to calculate.")
|
||||
parser.add_argument("--overhead", type=float, default=2.0, help="Estimated overhead in GiB for compute buffers, drivers, etc. (default: 2.0)")
|
||||
args = parser.parse_args()
|
||||
run_estimator(args.gguf_file, args.contexts, args.overhead)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h
|
||||
index 8b172e60..b813f523 100644
|
||||
--- a/ggml/src/ggml-cuda/vendors/hip.h
|
||||
+++ b/ggml/src/ggml-cuda/vendors/hip.h
|
||||
@@ -137,19 +137,11 @@
|
||||
#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR
|
||||
#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED
|
||||
|
||||
-#if HIP_VERSION >= 70000000
|
||||
-#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
|
||||
-#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
|
||||
+#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
|
||||
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
|
||||
#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F
|
||||
-#define cublasComputeType_t hipblasComputeType_t
|
||||
-#define cudaDataType_t hipDataType
|
||||
-#else
|
||||
-#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F
|
||||
-#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F
|
||||
-#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F
|
||||
-#define cublasComputeType_t hipblasDatatype_t
|
||||
-#define cudaDataType_t hipblasDatatype_t
|
||||
-#endif // HIP_VERSION >= 7000000
|
||||
+#define cublasComputeType_t hipblasComputeType_t
|
||||
+#define cudaDataType_t hipDataType
|
||||
|
||||
#if !defined(__HIP_PLATFORM_AMD__)
|
||||
#error "The HIP backend supports only AMD targets"
|
||||
Reference in New Issue
Block a user