first commit

benchmark/delete_transient_failures.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+import argparse
+import glob
+import os
+import re
+
+RESULTS_DIR_DEFAULT = "results"
+
+# Same detection logic as your extractor
+HEADER_RE = re.compile(r"^\|\s*model\s*\|", re.IGNORECASE)
+SEP_RE    = re.compile(r"^\|\s*-+")
+
+LOAD_ERR    = re.compile(r"failed to load model|Device memory allocation.*failed|⚠️\s*Fail", re.IGNORECASE)
+HANG_ERR    = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
+GENERIC_ERR = re.compile(r"error:|exit \d+|runtime error|⚠️\s*Runtime Error", re.IGNORECASE)
+
+
+def parse_table(text):
+    lines = text.splitlines()
+    rows = []
+    header = None
+    col_idx = {}
+
+    for line in lines:
+        if HEADER_RE.search(line):
+            header = [c.strip().lower() for c in line.strip().strip("|").split("|")]
+            for idx, name in enumerate(header):
+                col_idx[name] = idx
+            continue
+
+        if header and (SEP_RE.search(line) or not line.strip()):
+            continue
+
+        if header and line.startswith("|"):
+            parts = [c.strip() for c in line.strip().strip("|").split("|")]
+            if len(parts) < len(header):
+                continue
+            row = {}
+            for name, idx in col_idx.items():
+                row[name] = parts[idx]
+            rows.append(row)
+
+        if header and line.strip() == "" and rows:
+            break
+
+    return rows
+
+
+def detect_error(text):
+    if LOAD_ERR.search(text):
+        return True
+    if HANG_ERR.search(text):
+        return True
+    if GENERIC_ERR.search(text):
+        return True
+    return False
+
+
+def is_non_transient_vram_issue(text):
+    # Do NOT delete logs with this kind of Vulkan OOM
+    return (
+        "ggml_vulkan: Device memory allocation of size" in text
+        and "Requested buffer size exceeds device buffer size limit" in text
+    )
+
+
+def is_failed_run(text):
+    table_rows = parse_table(text)
+
+    has_pp = any(r.get("test", "").lower() == "pp512" for r in table_rows)
+    has_tg = any(r.get("test", "").lower() == "tg128" for r in table_rows)
+
+    if has_pp or has_tg:
+        return False
+
+    return detect_error(text)
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Delete transient-failure benchmark logs in results/"
+    )
+    ap.add_argument(
+        "--results-dir",
+        default=RESULTS_DIR_DEFAULT,
+        help="Directory containing *.log files (default: results)",
+    )
+    ap.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Only print what would be deleted",
+    )
+    args = ap.parse_args()
+
+    results_dir = args.results_dir
+    pattern = os.path.join(results_dir, "*.log")
+
+    to_delete = []
+    skipped_non_transient = []
+
+    for path in sorted(glob.glob(pattern)):
+        try:
+            with open(path, errors="ignore") as f:
+                text = f.read()
+        except OSError as e:
+            print(f"Could not read {path}: {e}")
+            continue
+
+        if not is_failed_run(text):
+            continue
+
+        if is_non_transient_vram_issue(text):
+            skipped_non_transient.append(path)
+            continue
+
+        to_delete.append(path)
+
+    if not to_delete and not skipped_non_transient:
+        print("No failed logs found.")
+        return
+
+    if skipped_non_transient:
+        print("Keeping logs with non transient VRAM issues:")
+        for p in skipped_non_transient:
+            print(f"  KEEP  {p}")
+
+    if to_delete:
+        print("Deleting logs with transient failures:")
+        for p in to_delete:
+            print(f"  DELETE {p}")
+            if not args.dry_run:
+                try:
+                    os.remove(p)
+                except OSError as e:
+                    print(f"    Failed to delete {p}: {e}")
+    else:
+        print("No logs to delete.")
+
+
+if __name__ == "__main__":
+    main()

benchmark/generate_markdown_results.py

@@ -0,0 +1,571 @@
+#!/usr/bin/env python3
+"""
+gen_benchmarks_md.py — Generate Markdown for README + detailed benchmarks from results.json
+
+Defaults:
+- Input JSON: ../docs/results.json
+- Outputs: ./README_benchmarks_section.md and ./benchmarks_generated.md
+"""
+
+from __future__ import annotations
+import json
+import argparse
+import statistics as stats
+from pathlib import Path
+from collections import defaultdict
+from typing import Dict, List, Tuple, Optional
+
+# === ENV LABELS ===
+ENV_LABEL: Dict[str, str] = {
+    # ROCm 7 RC
+    "rocm7_rc-rocwmma": "ROCm 7 RC + ROCWMMA + hipBLASLt",
+    "rocm7_rc": "ROCm 7 RC (hipBLASLt)",
+    "rocm7_rc-hblt0": "ROCm 7 RC (hipBLASLt OFF)",
+    "rocm7_rc-rocwmma-hblt0": "ROCm 7 RC + ROCWMMA (hipBLASLt OFF)",
+
+    # ROCm 6.4.4
+    "rocm6_4_4": "ROCm 6.4.4 (hipBLASLt)",
+    "rocm6_4_4-hblt0": "ROCm 6.4.4 (hipBLASLt OFF)",
+    "rocm6_4_4-rocwmma": "ROCm 6.4.4 + ROCWMMA (hipBLASLt)",
+    "rocm6_4_4-rocwmma-hblt0": "ROCm 6.4.4 + ROCWMMA (hipBLASLt OFF)",
+
+    # Vulkan
+    "vulkan_amdvlk": "Vulkan AMDVLK",
+    "vulkan_radv": "Vulkan RADV",
+}
+
+TESTS = ["pp512", "tg128"]
+
+def md_row(values: List[str]) -> str:
+    return "| " + " | ".join(values) + " |"
+
+
+def load_results(path: Path) -> Dict:
+    data = json.loads(path.read_text())
+    assert "runs" in data and isinstance(data["runs"], list), "results.json must have a top-level 'runs' list"
+    return data
+
+
+def envs_present(runs: List[Dict], only_env: Optional[List[str]], include_all_envs: bool) -> List[str]:
+    present = {r.get("env") for r in runs if r.get("env")}
+    if only_env:
+        present = present.intersection(set(only_env))
+    if include_all_envs:
+        # Include even if not present (might appear 0 rows in tables)
+        envs = [e for e in ENV_LABEL.keys() if (not only_env or e in only_env)]
+    else:
+        envs = [e for e in ENV_LABEL.keys() if e in present and (not only_env or e in only_env)]
+    return envs
+
+
+def fa_to_filter(fa: str) -> Optional[bool]:
+    fa = fa.lower().strip()
+    if fa == "on":
+        return True
+    if fa == "off":
+        return False
+    if fa == "any":
+        return None
+    raise ValueError("--fa must be on/off/any")
+
+
+def margin_aware_placements(
+    runs: List[Dict],
+    envs: List[str],
+    test_filter: str,
+    fa_filter: Optional[bool]
+) -> Tuple[Dict[str, Dict[str, int]], int]:
+    """
+    Returns (placements, sample_count)
+    placements[env] -> {"first": n, "second": n, "third": n}
+    sample_count = number of model+quant comparisons considered
+    """
+    placements = defaultdict(lambda: {"first": 0, "second": 0, "third": 0})
+    # group by (model, quant)
+    grouped = defaultdict(list)
+    for r in runs:
+        if r.get("error"):
+            continue
+        if r.get("test") != test_filter:
+            continue
+        if fa_filter is not None and r.get("fa") != fa_filter:
+            continue
+        if r.get("env") not in envs:
+            continue
+        key = (r.get("model_clean"), r.get("quant"))
+        grouped[key].append(r)
+
+    samples = 0
+    for key, entries in grouped.items():
+        # collate by env
+        env_groups = defaultdict(list)
+        for e in entries:
+            env_groups[e["env"]].append(e)
+        env_list = [e for e in envs if e in env_groups]  # keep requested order
+        if len(env_list) < 2:
+            continue
+
+        # summarize median mean ± median err per env
+        summary = {}
+        for env in env_list:
+            means = [x["tps_mean"] for x in env_groups[env] if x.get("tps_mean") is not None]
+            errs = [x.get("tps_err", 0.0) or 0.0 for x in env_groups[env]]
+            if not means:
+                continue
+            m = stats.median(means)
+            e = stats.median(errs) if errs else 0.0
+            summary[env] = (m - e, m + e, m)
+        if len(summary) < 2:
+            continue
+
+        samples += 1
+
+        # rank with overlap -> ties share rank
+        remaining = [env for env, _ in sorted(summary.items(), key=lambda kv: kv[1][2], reverse=True)]
+        assigned = {}
+        current_rank = 1
+        while remaining and current_rank <= 3:
+            env0 = remaining[0]
+            low0, high0, _ = summary[env0]
+            tied = [env0]
+            for env in remaining[1:]:
+                low, high, _ = summary[env]
+                if not (low > high0 or high < low0):  # overlap -> tie
+                    tied.append(env)
+            for env in tied:
+                assigned[env] = current_rank
+            remaining = [e for e in remaining if e not in tied]
+            current_rank += 1
+
+        for env, rk in assigned.items():
+            if rk == 1:
+                placements[env]["first"] += 1
+            elif rk == 2:
+                placements[env]["second"] += 1
+            elif rk == 3:
+                placements[env]["third"] += 1
+
+    return placements, samples
+
+
+def pairwise_win_counts(runs: List[Dict], envA: str, envB: str, test: str, fa_filter: Optional[bool]) -> Tuple[int, int, int, int]:
+    A = {}
+    B = {}
+    for r in runs:
+        if r.get("error") or r.get("test") != test:
+            continue
+        if fa_filter is not None and r.get("fa") != fa_filter:
+            continue
+        key = (r.get("model_clean"), r.get("quant"))
+        if r.get("env") == envA:
+            A[key] = r["tps_mean"]
+        elif r.get("env") == envB:
+            B[key] = r["tps_mean"]
+    winsA = winsB = ties = 0
+    for k in (set(A) & set(B)):
+        if A[k] > B[k]:
+            winsA += 1
+        elif B[k] > A[k]:
+            winsB += 1
+        else:
+            ties += 1
+    total = winsA + winsB + ties
+    return winsA, winsB, ties, total
+
+
+def average_ranks(place_dict: Dict[str, Dict[str, int]]) -> Dict[str, Optional[float]]:
+    avg = {}
+    for env, c in place_dict.items():
+        total = c.get("first", 0) + c.get("second", 0) + c.get("third", 0)
+        if total == 0:
+            avg[env] = None
+        else:
+            avg[env] = round((1 * c.get("first", 0) + 2 * c.get("second", 0) + 3 * c.get("third", 0)) / total, 2)
+    return avg
+
+
+def flash_attention_effect(runs: List[Dict], envs: List[str]) -> Dict[str, Dict[str, Dict[str, float]]]:
+    """
+    Returns: effects[env][test] = {n_pairs, median_pct, min, max}
+    Based on paired model+quant runs (ON vs OFF).
+    """
+    model_pairs = defaultdict(lambda: defaultdict(dict))  # (env,test)->(model,quant)->{fa: tps}
+    for r in runs:
+        if r.get("error") or r.get("tps_mean") is None:
+            continue
+        if r.get("test") not in TESTS:
+            continue
+        if r.get("env") not in envs:
+            continue
+        model_key = (r.get("model_clean"), r.get("quant"))
+        model_pairs[(r["env"], r["test"])][model_key][r.get("fa")] = r["tps_mean"]
+
+    summary = defaultdict(dict)
+    for (env, test), d in model_pairs.items():
+        deltas = []
+        for mk, vals in d.items():
+            if True in vals and False in vals and vals[False] > 0:
+                deltas.append((vals[True] - vals[False]) / vals[False] * 100.0)
+        if deltas:
+            summary[env][test] = {
+                "n_pairs": len(deltas),
+                "median_pct": round(stats.median(deltas), 1),
+                "min": round(min(deltas), 1),
+                "max": round(max(deltas), 1),
+            }
+    return summary
+
+
+def rocwmma_effect(runs: List[Dict], pairs_to_compare: List[Tuple[str, str, str]], tests: List[str]) -> List[Tuple[str, str, str, str, int, float]]:
+    """
+    Compare ROCWMMA ON vs OFF with same hipBLASLt state.
+    Returns rows of (context_label, test, env_on, env_off, n_pairs, median_delta_pct)
+    where delta_pct = median(ON/OFF - 1)*100 over common model+quant.
+    """
+    rows = []
+    for env_on, env_off, label in pairs_to_compare:
+        for test in tests:
+            data_on = defaultdict(list)
+            data_off = defaultdict(list)
+            for r in runs:
+                if r.get("error") or r.get("test") != test:
+                    continue
+                if r.get("env") == env_on:
+                    data_on[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
+                elif r.get("env") == env_off:
+                    data_off[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
+            common = sorted(set(data_on) & set(data_off))
+            if not common:
+                continue
+            ratios = []
+            for k in common:
+                aon = stats.median(data_on[k])
+                aoff = stats.median(data_off[k])
+                if aoff > 0:
+                    ratios.append(aon / aoff - 1.0)
+            if ratios:
+                rows.append((label, test, env_on, env_off, len(ratios), round(100 * stats.median(ratios), 1)))
+    return rows
+
+
+def hipblaslt_effect(runs: List[Dict], pairs_to_compare: List[Tuple[str, str, str]], tests: List[str]) -> List[Tuple[str, str, str, str, int, float]]:
+    """
+    Compare hipBLASLt ON vs OFF with same ROCWMMA state.
+    Returns rows of (context_label, test, env_on, env_off, n_pairs, median_delta_pct)
+    where delta_pct = median(ON/OFF - 1)*100 over common model+quant.
+    """
+    rows = []
+    for env_on, env_off, label in pairs_to_compare:
+        for test in tests:
+            data_on = defaultdict(list)
+            data_off = defaultdict(list)
+            for r in runs:
+                if r.get("error") or r.get("test") != test:
+                    continue
+                if r.get("env") == env_on:
+                    data_on[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
+                elif r.get("env") == env_off:
+                    data_off[(r.get("model_clean"), r.get("quant"))].append(r["tps_mean"])
+            common = sorted(set(data_on) & set(data_off))
+            if not common:
+                continue
+            ratios = []
+            for k in common:
+                aon = stats.median(data_on[k])
+                aoff = stats.median(data_off[k])
+                if aoff > 0:
+                    ratios.append(aon / aoff - 1.0)
+            if ratios:
+                rows.append((label, test, env_on, env_off, len(ratios), round(100 * stats.median(ratios), 1)))
+    return rows
+
+
+def amdvlk_vs_radv(runs: List[Dict], fa_filter: Optional[bool]) -> List[Tuple[str, int, int, int, int]]:
+    rows = []
+    for test in TESTS:
+        wa, wr, ties, total = pairwise_win_counts(runs, "vulkan_amdvlk", "vulkan_radv", test, fa_filter)
+        rows.append((test, wa, wr, ties, total))
+    return rows
+
+
+def winners(place_dict: Dict[str, Dict[str, int]], slot="first") -> Tuple[List[str], int]:
+    max_count = max((c.get(slot, 0) for c in place_dict.values()), default=0)
+    win_list = [env for env, c in place_dict.items() if c.get(slot, 0) == max_count and max_count > 0]
+    return win_list, max_count
+
+
+def human_list(envs: List[str]) -> str:
+    return ", ".join(ENV_LABEL.get(e, e) for e in envs) if envs else "—"
+
+
+def build_readme_section(
+    envs: List[str],
+    pp_place: Dict[str, Dict[str, int]],
+    tg_place: Dict[str, Dict[str, int]],
+    fa_filter: Optional[bool]
+) -> str:
+    # Winners
+    pp_wins, _ = winners(pp_place, "first")
+    tg_wins, _ = winners(tg_place, "first")
+
+    lines: List[str] = []
+    lines.append("## 3. Performance Benchmarks (Key Results)")
+    lines.append("")
+    lines.append("🌐 Interactive exploration of the latest benchmark runs: [Interactie Benchmark Viewer](https://kyuz0.github.io/amd-strix-halo-toolboxes/)")
+    lines.append("")
+    lines.append("Benchmarks were analysed with **error-aware ties** (mean ± σ). If two backends overlap within margins, they are treated as a tie. All placement counts below use **Flash Attention ON**.")
+    lines.append("")
+
+    # Placement tables
+    def place_table(title: str, place_dict: Dict[str, Dict[str, int]]):
+        lines.append(f"**{title}**")
+        lines.append(md_row(["Backend", "1st", "2nd", "3rd"]))
+        lines.append(md_row(["---", "---:", "---:", "---:"]))
+        order = sorted(place_dict.items(), key=lambda kv: (-kv[1].get("first", 0), -kv[1].get("second", 0), kv[0]))
+        for env, c in order:
+            lines.append(md_row([ENV_LABEL.get(env, env), str(c.get("first", 0)), str(c.get("second", 0)), str(c.get("third", 0))]))
+        lines.append("")
+
+    place_table("Prompt Processing (pp512)", pp_place)
+    place_table("Token Generation (tg128)", tg_place)
+
+    # Data-driven recommendations
+    def total_score(c: Dict[str, int]) -> int:
+        # weight 1st more than 2nd
+        return c.get("first", 0) * 2 + c.get("second", 0)
+
+    best_bal_score = -1
+    balanced: List[str] = []
+    for env in envs:
+        score = total_score(pp_place.get(env, {})) + total_score(tg_place.get(env, {}))
+        if score > best_bal_score:
+            best_bal_score = score
+            balanced = [env]
+        elif score == best_bal_score:
+            balanced.append(env)
+
+    lines.append("### Summary & Recommendations")
+    lines.append(f"- **Fastest prompt processing:** {human_list(pp_wins)} (most 1st-place finishes).")
+    lines.append(f"- **Fastest token generation:** {human_list(tg_wins)} (most 1st-place finishes).")
+    lines.append(f"- **Balanced choice:** {human_list(balanced)} (consistently near the top across PP/TG).")
+    lines.append("")
+    lines.append("> **Note (ROCm 7):** Toolboxes enable **hipBLASLt** by default. The benchmark suite also runs **hipBLASLt OFF** variants to show its impact.")
+    return "\n".join(lines)
+
+
+def build_benchmarks_doc(
+    runs: List[Dict],
+    envs: List[str],
+    pp_place: Dict[str, Dict[str, int]],
+    tg_place: Dict[str, Dict[str, int]],
+    fa_filter: Optional[bool],
+) -> str:
+    lines: List[str] = []
+    lines.append("# AMD Strix Halo — llama.cpp Toolboxes (Benchmarks)")
+    lines.append("")
+    lines.append("**Interactive results:** https://kyuz0.github.io/amd-strix-halo-toolboxes/")
+    lines.append("")
+    lines.append("## Table of Contents")
+    lines.append("- [Benchmark methodology](#benchmark-methodology)")
+    lines.append("- [Summary of current dataset (Flash Attention ON)](#summary-of-current-dataset-flash-attention-on)")
+    lines.append("  - [Placement counts](#placement-counts)")
+    lines.append("  - [Pairwise head-to-head wins](#pairwise-head-to-head-wins)")
+    lines.append("  - [Average ranks](#average-ranks)")
+    lines.append("- [Analyses by feature](#analyses-by-feature)")
+    lines.append("  - [Impact of Flash Attention](#impact-of-flash-attention)")
+    lines.append("  - [Impact of ROCWMMA](#impact-of-rocwmma)")
+    lines.append("  - [Impact of hipBLASLt](#impact-of-hipblaslt)")
+    lines.append("  - [Vulkan: AMDVLK vs RADV](#vulkan-amdvlk-vs-radv)")
+    lines.append("- [Recommendations](#recommendations)")
+    lines.append("- [Winner calculation](#winner-calculation)")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Benchmark methodology")
+    lines.append("")
+    lines.append("- **pp512** — prompt processing throughput (tokens/sec, prefill)")
+    lines.append("- **tg128** — token generation throughput (tokens/sec, interactive)")
+    lines.append("- Each backend tested twice per model: `-fa 0` and `-fa 1`")
+    lines.append("- Winners per model/test are **margin-aware**; multiple winners are possible when mean±σ overlap")
+    lines.append("- Built from the same llama.cpp commit for consistency")
+    lines.append("")
+    lines.append("**Backends in this dataset:** " + ", ".join(ENV_LABEL.get(e, e) for e in envs))
+    lines.append("")
+    lines.append("**ROCm 7 hipBLASLt policy:** Toolboxes ship with **hipBLASLt enabled** by default (`ROCBLAS_USE_HIPBLASLT=1`). The benchmark script also runs **hipBLASLt OFF** variants (`-hblt0`) to measure its effect.")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Summary of current dataset (Flash Attention ON)")
+    lines.append("")
+    # Placement counts
+    lines.append("### Placement counts")
+    def place_block(title: str, place_dict: Dict[str, Dict[str, int]]):
+        lines.append(f"**{title}**")
+        lines.append(md_row(["Backend", "1st", "2nd", "3rd"]))
+        lines.append(md_row(["---", "---:", "---:", "---:"]))
+        order = sorted(place_dict.items(), key=lambda kv: (-kv[1].get("first", 0), -kv[1].get("second", 0), kv[0]))
+        for env, c in order:
+            lines.append(md_row([ENV_LABEL.get(env, env), str(c.get("first", 0)), str(c.get("second", 0)), str(c.get("third", 0))]))
+        lines.append("")
+    place_block("Prompt Processing (pp512)", pp_place)
+    place_block("Token Generation (tg128)", tg_place)
+
+    # Pairwise wins
+    lines.append("### Pairwise head-to-head wins")
+    lines.append("For any model+quant where both backends succeeded, this counts who was faster (ties when equal).")
+    lines.append(md_row(["Comparison", "Test", "A wins", "B wins", "Ties", "Total"]))
+    lines.append(md_row(["---", "---", "---:", "---:", "---:", "---:"]))
+    pairs = [
+        ("ROCm 7 RC + ROCWMMA + hipBLASLt", "Vulkan AMDVLK", "rocm7_rc-rocwmma", "vulkan_amdvlk"),
+        ("ROCm 7 RC + ROCWMMA + hipBLASLt", "Vulkan RADV", "rocm7_rc-rocwmma", "vulkan_radv"),
+        ("Vulkan AMDVLK", "Vulkan RADV", "vulkan_amdvlk", "vulkan_radv"),
+    ]
+    for labelA, labelB, envA, envB in pairs:
+        for test in TESTS:
+            a, b, t, total = pairwise_win_counts(runs, envA, envB, test, fa_filter)
+            lines.append(md_row([f"{labelA} vs {labelB}", test, str(a), str(b), str(t), str(total)]))
+    lines.append("")
+
+    # Average ranks
+    lines.append("### Average ranks")
+    avg_pp = average_ranks(pp_place)
+    avg_tg = average_ranks(tg_place)
+    lines.append("**Prompt Processing (pp512)**")
+    lines.append(md_row(["Backend", "Avg Rank (↓ is better)"]))
+    lines.append(md_row(["---", "---:"]))
+    for env, val in sorted(avg_pp.items(), key=lambda kv: (kv[1] is None, kv[1] or 99)):
+        lines.append(md_row([ENV_LABEL.get(env, env), str(val) if val is not None else "—"]))
+    lines.append("")
+    lines.append("**Token Generation (tg128)**")
+    lines.append(md_row(["Backend", "Avg Rank (↓ is better)"]))
+    lines.append(md_row(["---", "---:"]))
+    for env, val in sorted(avg_tg.items(), key=lambda kv: (kv[1] is None, kv[1] or 99)):
+        lines.append(md_row([ENV_LABEL.get(env, env), str(val) if val is not None else "—"]))
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Analyses by feature")
+    lines.append("")
+
+    # Flash Attention effect
+    lines.append("### Impact of Flash Attention")
+    fa_eff = flash_attention_effect(runs, envs)
+    lines.append("Median % change when **Flash Attention ON vs OFF**, paired by model+quant, per backend:")
+    lines.append(md_row(["Backend", "pp512 Δ% (median, min..max, n)", "tg128 Δ% (median, min..max, n)"]))
+    lines.append(md_row(["---", "---", "---"]))
+    def fmt_eff(row: Optional[Dict[str, float]]) -> str:
+        return f"{row['median_pct']}% ({row['min']}..{row['max']}), n={row['n_pairs']}" if row else "—"
+    for env in envs:
+        row_pp = fa_eff.get(env, {}).get("pp512")
+        row_tg = fa_eff.get(env, {}).get("tg128")
+        lines.append(md_row([ENV_LABEL.get(env, env), fmt_eff(row_pp), fmt_eff(row_tg)]))
+    lines.append("")
+
+    # ROCWMMA effect — check both ROCm 7 and 6.4.4 families if present
+    lines.append("### Impact of ROCWMMA")
+    rocwmma_pairs = []
+    if "rocm7_rc-rocwmma" in envs and "rocm7_rc" in envs:
+        rocwmma_pairs.append(("rocm7_rc-rocwmma", "rocm7_rc", "ROCm 7 RC (hipBLASLt)"))
+    if "rocm7_rc-rocwmma-hblt0" in envs and "rocm7_rc-hblt0" in envs:
+        rocwmma_pairs.append(("rocm7_rc-rocwmma-hblt0", "rocm7_rc-hblt0", "ROCm 7 RC (hipBLASLt OFF)"))
+    if "rocm6_4_4-rocwmma" in envs and "rocm6_4_4" in envs:
+        rocwmma_pairs.append(("rocm6_4_4-rocwmma", "rocm6_4_4", "ROCm 6.4.4 (hipBLASLt)"))
+    if "rocm6_4_4-rocwmma-hblt0" in envs and "rocm6_4_4-hblt0" in envs:
+        rocwmma_pairs.append(("rocm6_4_4-rocwmma-hblt0", "rocm6_4_4-hblt0", "ROCm 6.4.4 (hipBLASLt OFF)"))
+
+    rocwmma_rows = rocwmma_effect(runs, rocwmma_pairs, TESTS)
+    lines.append(md_row(["Context", "Test", "Compared Envs", "Pairs", "Median Δ%"]))
+    lines.append(md_row(["---", "---", "---", "---:", "---:"]))
+    for label, test, env_on, env_off, n, delta in rocwmma_rows:
+        lines.append(md_row([label, test, f"{ENV_LABEL.get(env_on, env_on)} vs {ENV_LABEL.get(env_off, env_off)}", str(n), f"{delta}%"]))
+    lines.append("")
+
+    # hipBLASLt effect — for both ROCm 7 and 6.4.4 families
+    lines.append("### Impact of hipBLASLt")
+    hip_pairs = []
+    if "rocm7_rc" in envs and "rocm7_rc-hblt0" in envs:
+        hip_pairs.append(("rocm7_rc", "rocm7_rc-hblt0", "ROCm 7 RC (no ROCWMMA)"))
+    if "rocm7_rc-rocwmma" in envs and "rocm7_rc-rocwmma-hblt0" in envs:
+        hip_pairs.append(("rocm7_rc-rocwmma", "rocm7_rc-rocwmma-hblt0", "ROCm 7 RC + ROCWMMA"))
+    if "rocm6_4_4" in envs and "rocm6_4_4-hblt0" in envs:
+        hip_pairs.append(("rocm6_4_4", "rocm6_4_4-hblt0", "ROCm 6.4.4 (no ROCWMMA)"))
+    if "rocm6_4_4-rocwmma" in envs and "rocm6_4_4-rocwmma-hblt0" in envs:
+        hip_pairs.append(("rocm6_4_4-rocwmma", "rocm6_4_4-rocwmma-hblt0", "ROCm 6.4.4 + ROCWMMA"))
+
+    hip_rows = hipblaslt_effect(runs, hip_pairs, TESTS)
+    lines.append(md_row(["Context", "Test", "Compared Envs", "Pairs", "Median Δ%"]))
+    lines.append(md_row(["---", "---", "---", "---:", "---:"]))
+    for label, test, env_on, env_off, n, delta in hip_rows:
+        lines.append(md_row([label, test, f"{ENV_LABEL.get(env_on, env_on)} vs {ENV_LABEL.get(env_off, env_off)}", str(n), f"{delta}%"]))
+    lines.append("")
+
+    # AMDVLK vs RADV
+    lines.append("### Vulkan: AMDVLK vs RADV")
+    lines.append("Head-to-head wins with selected Flash Attention filter:")
+    lines.append(md_row(["Test", "AMDVLK wins", "RADV wins", "Ties", "Total"]))
+    lines.append(md_row(["---", "---:", "---:", "---:", "---:"]))
+    for test, wa, wr, t, total in amdvlk_vs_radv(runs, fa_filter):
+        lines.append(md_row([test, str(wa), str(wr), str(t), str(total)]))
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Recommendations")
+    pp_wins, _ = winners(pp_place, "first")
+    tg_wins, _ = winners(tg_place, "first")
+    lines.append(f"- **Fastest prompt processing:** {human_list(pp_wins)} (most 1st-place finishes with selected Flash Attention filter).")
+    lines.append(f"- **Fastest token generation:** {human_list(tg_wins)} (most 1st-place finishes with selected Flash Attention filter).")
+    # Balanced: highest (2*first + second) across PP+TG
+    def score(c: Dict[str, int]) -> int:
+        return c.get("first", 0) * 2 + c.get("second", 0)
+    best_bal = -1
+    balanced: List[str] = []
+    for env in envs:
+        s = score(pp_place.get(env, {})) + score(tg_place.get(env, {}))
+        if s > best_bal:
+            best_bal = s
+            balanced = [env]
+        elif s == best_bal:
+            balanced.append(env)
+    lines.append(f"- **Balanced choice:** {human_list(balanced)} (consistently near the top across PP/TG).")
+    lines.append("")
+    lines.append("---")
+    lines.append("")
+    lines.append("## Winner calculation")
+    lines.append("A backend is counted as a winner if its mean throughput is within the best backend’s pooled ± error margin for that model/test type. This treats results within measurement noise as ties instead of false losses.")
+    return "\n".join(lines)
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--file", type=Path, default=Path("../docs/results.json"),
+                    help="Path to results.json (default: ../docs/results.json)")
+    ap.add_argument("--out-readme", type=Path, default=Path("./README_benchmarks_section.md"),
+                    help="Path to write README section Markdown (default: ./README_benchmarks_section.md)")
+    ap.add_argument("--out-bench", type=Path, default=Path("./benchmarks_generated.md"),
+                    help="Path to write detailed benchmarks Markdown (default: ./benchmarks_generated.md)")
+    ap.add_argument("--fa", choices=["on", "off", "any"], default="on",
+                    help="Flash Attention filter (default: on)")
+    ap.add_argument("--include-all-envs", action="store_true",
+                    help="Include envs even if not present in results.json")
+    ap.add_argument("--only-env", action="append",
+                    help="Restrict analysis to specific env keys (repeatable)")
+    args = ap.parse_args()
+
+    data = load_results(args.file)
+    runs: List[Dict] = data["runs"]
+    fa_filter = fa_to_filter(args.fa)
+    envs = envs_present(runs, args.only_env, args.include_all_envs)
+
+    pp_place, _ = margin_aware_placements(runs, envs, "pp512", fa_filter)
+    tg_place, _ = margin_aware_placements(runs, envs, "tg128", fa_filter)
+
+    readme_md = build_readme_section(envs, pp_place, tg_place, fa_filter)
+    args.out_readme.write_text(readme_md)
+
+    bench_md = build_benchmarks_doc(runs, envs, pp_place, tg_place, fa_filter)
+    args.out_bench.write_text(bench_md)
+
+    print(f"Wrote:\n - {args.out_readme}\n - {args.out_bench}")
+
+
+if __name__ == "__main__":
+    main()

benchmark/generate_results.json.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+import re, glob, os, json, time
+from pathlib import Path
+
+RESULT_SOURCES = [
+    ("results", False),       # regular single-node runs
+    ("results-rpc", True),    # distributed RPC runs across two servers
+]
+OUT_JSON = "../docs/results.json"
+
+# --- Regexes ---------------------------------------------------------------
+
+# Table headers come in two shapes (with or without "fa" column)
+HEADER_RE = re.compile(r"^\|\s*model\s*\|", re.IGNORECASE)
+SEP_RE    = re.compile(r"^\|\s*-+")
+
+# Build line, e.g. "build: cd6983d5 (6119)"
+BUILD_RE  = re.compile(r"build:\s*([0-9a-f]{7,})\s*\((\d+)\)", re.IGNORECASE)
+
+# Error classifiers (same spirit as your table script)
+LOAD_ERR   = re.compile(r"failed to load model|Device memory allocation.*failed|⚠️\s*Fail", re.IGNORECASE)
+HANG_ERR   = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
+GENERIC_ERR= re.compile(r"error:|exit \d+|runtime error|⚠️\s*Runtime Error", re.IGNORECASE)
+
+# Extract numeric ± numeric from the last column
+TS_RE      = re.compile(r"([\d.]+)\s*±\s*([\d.]+)")
+
+# Quantization from model name
+QUANT_RE = re.compile(r"(Q\d+_[A-Z0-9_]+|BF16|F16|F32|mxfp\d+)", re.IGNORECASE)
+
+PARAMS_RE = re.compile(r"([\d.,]+)\s*B", re.IGNORECASE)
+GIB_RE    = re.compile(r"([\d.,]+)\s*GiB", re.IGNORECASE)
+
+# "30B", "235B" from model name
+NAME_B_RE = re.compile(r"(\d+(?:\.\d+)?)B")
+
+# Shard suffix in filenames
+SHARD_RE = re.compile(r"-000\d+-of-000\d+", re.IGNORECASE)
+
+# Long-context suffix in filenames (e.g., __longctx32768)
+LONGCTX_RE = re.compile(r"longctx(\d+)", re.IGNORECASE)
+
+# --- Helpers ---------------------------------------------------------------
+
+ENV_CANON = {
+    "rocm7_1": "rocm7.1",
+    "rocm7_alpha": "rocm-7alpha",
+}
+
+def clean_model_name(raw):
+    base = SHARD_RE.sub("", raw)
+    return base
+
+def canonicalize_env(env):
+    if not env:
+        return env
+    for raw, canon in ENV_CANON.items():
+        prefix = f"{raw}-"
+        if env == raw:
+            return canon
+        if env.startswith(prefix):
+            return canon + env[len(raw):]
+    return env
+
+def parse_env_flags(basename):
+    """
+    pattern: <model>__<env>[__fa1][__hblt0][__longctx32768][__rpc]
+    Returns (env, fa, context_tag, context_tokens, rpc_flag)
+    """
+    parts = basename.split("__")
+    if len(parts) < 2:
+        return None, False, "default", None, False
+
+    env = parts[1]
+    fa = False
+    context_tag = "default"
+    context_tokens = None
+    rpc_flag = False
+
+    for raw_suffix in parts[2:]:
+        suffix = raw_suffix.lower()
+        if suffix == "fa1":
+            fa = True
+        elif suffix == "hblt0":
+            env = f"{env}-hblt0"
+        elif suffix.startswith("longctx"):
+            context_tag = suffix
+            m = LONGCTX_RE.search(suffix)
+            if m:
+                try:
+                    context_tokens = int(m.group(1))
+                except ValueError:
+                    context_tokens = None
+        elif suffix == "rpc":
+            rpc_flag = True
+
+    return env, fa, context_tag, context_tokens, rpc_flag
+
+def env_base_and_variant(env):
+    # e.g. "rocm6_4_2-rocwmma" -> ("rocm6_4_2", "rocwmma")
+    if "-" in env:
+        base, variant = env.split("-", 1)
+        return base, variant
+    return env, None
+
+def detect_error(text):
+    if LOAD_ERR.search(text):
+        return True, "load"
+    if HANG_ERR.search(text):
+        return True, "hang"
+    if GENERIC_ERR.search(text):
+        return True, "runtime"
+    return False, None
+
+def parse_table(text):
+    """
+    Returns list of rows parsed from the markdown-like table.
+    Each row is a dict of the parsed columns, normalized by header names.
+    Handles presence/absence of the 'fa' column.
+    """
+    lines = text.splitlines()
+    rows = []
+    header = None
+    col_idx = {}
+
+    for i, line in enumerate(lines):
+        if HEADER_RE.search(line):
+            # header line
+            header = [c.strip().lower() for c in line.strip().strip("|").split("|")]
+            # next line should be the separator; skip it
+            # build index map
+            for idx, name in enumerate(header):
+                col_idx[name] = idx
+            continue
+        if header and (SEP_RE.search(line) or not line.strip()):
+            # skip separators / blanks after header
+            continue
+        if header and line.startswith("|"):
+            parts = [c.strip() for c in line.strip().strip("|").split("|")]
+            # guard for short lines
+            if len(parts) < len(header):
+                continue
+            row = {}
+            for name, idx in col_idx.items():
+                row[name] = parts[idx]
+            rows.append(row)
+        # stop parsing block when a blank line after some rows appears
+        if header and line.strip() == "" and rows:
+            break
+
+    return rows
+
+def coerce_float(m, default=None):
+    try:
+        return float(m)
+    except:
+        return default
+
+def extract_quant(model_name):
+    m = QUANT_RE.search(model_name)
+    return (m.group(1).upper() if m else None)
+
+def b_from_name(model_name):
+    m = NAME_B_RE.search(model_name)
+    return coerce_float(m.group(1)) if m else None
+
+# --- Main scan -------------------------------------------------------------
+
+runs = []
+builds = set()
+envs  = set()
+
+for results_dir, is_rpc_source in RESULT_SOURCES:
+    glob_pattern = os.path.join(results_dir, "*.log")
+    for path in sorted(glob.glob(glob_pattern)):
+        base = os.path.basename(path).rsplit(".log", 1)[0]
+        if "__" not in base:
+            continue
+
+        model_raw, _rest = base.split("__", 1)
+        env, fa_from_name, context_tag, context_tokens, rpc_flag = parse_env_flags(base)
+        env = canonicalize_env(env)
+        if env:
+            envs.add(env)
+
+        model_clean = clean_model_name(model_raw)
+
+        with open(path, errors="ignore") as f:
+            text = f.read()
+
+        # build info (take the last match in file if many)
+        build_hash, build_num = None, None
+        for m in BUILD_RE.finditer(text):
+            build_hash, build_num = m.group(1), m.group(2)
+        if build_hash:
+            builds.add((build_hash, build_num))
+
+        # detect error (if there is no valid table rows)
+        table_rows = parse_table(text)
+
+        # If table rows exist, we’ll still mark errors only if no perf found
+        has_pp = any(r.get("test","").lower()=="pp512" for r in table_rows)
+        has_tg = any(r.get("test","").lower()=="tg128" for r in table_rows)
+        error, etype = (False, None)
+        if not (has_pp or has_tg):
+            error, etype = detect_error(text)
+
+        # Determine FA flag:
+        #   prefer explicit column "fa" if present, else fallback to filename "__fa1"
+        fa_in_table = None
+        for r in table_rows:
+            if "fa" in r:
+                try:
+                    fa_in_table = int(r["fa"]) == 1
+                except:
+                    fa_in_table = None
+                break
+        fa_enabled = fa_in_table if fa_in_table is not None else fa_from_name
+
+        # Normalize env base / variant (e.g., rocwmma)
+        env_base, env_variant = env_base_and_variant(env)
+
+        # Emit one run per row (pp512 / tg128)
+        for r in table_rows or [{}]:
+            test = r.get("test", "").lower() if table_rows else None
+            tps_mean, tps_std = None, None
+            if table_rows:
+                ts_field = r.get("t/s", "")
+                m = TS_RE.search(ts_field)
+                if m:
+                    tps_mean = coerce_float(m.group(1))
+                    tps_std  = coerce_float(m.group(2))
+
+            # parse numeric helpers from row (if present)
+            params_b = None
+            file_size_gib = None
+            if "params" in r:
+                pm = PARAMS_RE.search(r["params"])
+                if pm:
+                    params_b = coerce_float(pm.group(1).replace(",", ""))
+            if "size" in r:
+                sm = GIB_RE.search(r["size"])
+                if sm:
+                    file_size_gib = coerce_float(sm.group(1).replace(",", ""))
+
+            # quant from model name (unchanged)
+            quant = extract_quant(model_clean)
+
+            # name_params_b: prefer table value; else fall back to B in model name
+            name_params_b = params_b if params_b is not None else b_from_name(model_clean)
+
+            backend = r.get("backend")
+            ngl = r.get("ngl")
+            mmap = r.get("mmap")
+
+            run = {
+                "model": model_raw,
+                "model_clean": model_clean,
+                "env": env,
+                "env_base": env_base,
+                "env_variant": env_variant,         # e.g. "rocwmma"
+                "fa": bool(fa_enabled),
+                "context": context_tag or "default",
+                "context_tokens": context_tokens,
+                "test": test,                       # "pp512" | "tg128" | None (if error)
+                "tps_mean": tps_mean,
+                "tps_std": tps_std,
+                "error": bool(error),
+                "error_type": etype,                # "load" | "hang" | "runtime" | None
+                "backend": backend,
+                "ngl": (int(ngl) if (ngl and ngl.isdigit()) else None),
+                "mmap": (int(mmap) if (mmap and mmap.isdigit()) else None),
+                "params_b": params_b,               # from table, if available
+                "file_size_gib": file_size_gib,     # from table, if available
+                "name_params_b": name_params_b,     # parsed from model name (e.g., 30B -> 30.0)
+                "quant": quant,
+                "log": path,
+                "rpc": bool(is_rpc_source or rpc_flag),
+                "build": {"hash": build_hash, "number": build_num} if build_hash else None,
+            }
+            runs.append(run)
+
+# Meta
+meta = {
+    "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+    "os_kernel": "Fedora 42 — Linux 6.15.9-201.fc42.x86_64 (Sat Aug  2 11:37:34 UTC 2025)",
+    "llamacpp_builds": [{"hash": h, "number": n} for (h, n) in sorted(builds)],
+    "environments": sorted(envs),
+    "notes": "pp512 = prompt processing; tg128 = text generation; t/s = tokens/second",
+}
+
+out = {"meta": meta, "runs": runs}
+
+Path(OUT_JSON).write_text(json.dumps(out, indent=2))
+print(f"Wrote {OUT_JSON} with {len(runs)} rows.")

benchmark/parse_benchmark_results.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+import re, glob, os
+
+# This script parses llama-bench logs in 'results/' to produce
+# Markdown tables for pp512 (prompt processing) and tg128 (text generation).
+
+# Regex patterns to extract tokens/sec rows
+PP_RE = re.compile(r"\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|\s*pp512\s*\|\s*([\d.]+)\s*±\s*([\d.]+)")
+TG_RE = re.compile(r"\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|[^|]*\|\s*tg128\s*\|\s*([\d.]+)\s*±\s*([\d.]+)")
+
+# Patterns to classify errors
+LOAD_ERR = re.compile(r"failed to load model|Device memory allocation.*failed", re.IGNORECASE)
+HANG_ERR = re.compile(r"GPU Hang|HW Exception", re.IGNORECASE)
+GENERIC_ERR = re.compile(r"error:|exit \d+", re.IGNORECASE)
+
+# Env ordering
+ENV_ORDER = ["vulkan_radv","vulkan_amdvlk","rocm6_4_2","rocm7_beta","rocm7_rc"]
+
+data = {}
+
+# Utility to clean model names
+def clean_name(raw):
+    return re.sub(r"-000\d+-of-000\d+", "", raw)
+
+# Scan logs
+glob_pattern = os.path.join("results", "*.log")
+for path in sorted(glob.glob(glob_pattern)):
+    # Fix: use rsplit, not rssplit
+    base = os.path.basename(path).rsplit('.log',1)[0]
+    if '__' not in base:
+        continue
+    model_raw, env = base.split('__',1)
+    model = clean_name(model_raw)
+
+    text = open(path, errors='ignore').read()
+    # Determine error type
+    if LOAD_ERR.search(text):
+        err_type = 'load'
+    elif HANG_ERR.search(text):
+        err_type = 'hang'
+    elif GENERIC_ERR.search(text) and not (PP_RE.search(text) and TG_RE.search(text)):
+        err_type = 'runtime'
+    else:
+        err_type = None
+
+    # Extract performance if no load error
+    pp_match = PP_RE.search(text) if err_type is None else None
+    tg_match = TG_RE.search(text) if err_type is None else None
+
+    for key, match in [('pp512', pp_match), ('tg128', tg_match)]:
+        cell = {
+            'mean': match.group(1) if match else None,
+            'std':  match.group(2) if match else None,
+            'error': err_type is not None,
+            'etype': err_type
+        }
+        data.setdefault(model, {}).setdefault(key, {})[env] = cell
+
+# Select winner
+def pick_winner(env_data):
+    scores = {e: float(d['mean']) for e,d in env_data.items() if not d['error'] and d['mean']}
+    if not scores:
+        return '—'
+    best = max(scores, key=scores.get)
+    others = [v for k,v in scores.items() if k!=best]
+    tag = f"🏆 **{best}**"
+    if others:
+        gain = (scores[best]/max(others)-1)*100
+        tag += f" (+{gain:.0f}%)"
+    return tag
+
+# Render table with distinct error messages
+def render_table(test_label, display_name):
+    print(f"### {display_name} — tokens/second\n")
+    header = ['Model'] + [e.replace('_',' ').title() for e in ENV_ORDER] + ['Winner']
+    print("| " + " | ".join(header) + " |")
+    print("|" + "|".join(['---']*len(header)) + "|")
+
+    for model in sorted(data, key=lambda s: s.lower()):
+        row = [f"**{model}**"]
+        env_data = data[model].get(test_label, {})
+        for env in ENV_ORDER:
+            d = env_data.get(env)
+            if not d:
+                cell = '—'
+            elif d['error']:
+                et = d['etype']
+                if et=='load':
+                    cell = '⚠️ Load Error'
+                elif et=='hang':
+                    cell = '⚠️ GPU Hang'
+                else:
+                    cell = '⚠️ Runtime Error'
+            else:
+                cell = f"{float(d['mean']):.2f} ± {float(d['std']):.2f}"
+            row.append(cell)
+        row.append(pick_winner(env_data))
+        print("| " + " | ".join(row) + " |")
+    print()
+
+# Output tables
+render_table('pp512','Prompt Processing (pp512)')
+render_table('tg128','Text Generation (tg128)')
+
+# Summary of failures by type
+fail_lines = []
+for model in sorted(data, key=lambda s: s.lower()):
+    for test_label, envs in data[model].items():
+        for env,d in envs.items():
+            if d['error']:
+                et = d['etype'] or 'unknown'
+                desc = {
+                    'load':'failed to load',
+                    'hang':'GPU hang',
+                    'runtime':'runtime error',
+                }.get(et, 'error')
+                fail_lines.append(f"- **{model}** [{test_label}] on *{env}*: {desc}")
+if fail_lines:
+    print("## Failed Runs\n")
+    print("\n".join(fail_lines))    

benchmark/run_benchmarks.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+MODEL_DIR="$(realpath models)"
+RESULTDIR="results"
+mkdir -p "$RESULTDIR"
+
+# Pick exactly one .gguf per model: either
+#  - any .gguf without "-000*-of-" (single-file models)
+#  - or the first shard "*-00001-of-*.gguf"
+mapfile -t MODEL_PATHS < <(
+  find "$MODEL_DIR" -type f -name '*.gguf' \
+    \( -name '*-00001-of-*.gguf' -o -not -name '*-000*-of-*.gguf' \) \
+    | sort
+)
+
+if (( ${#MODEL_PATHS[@]} == 0 )); then
+  echo "❌ No models found under $MODEL_DIR – check your paths/patterns!"
+  exit 1
+fi
+
+echo "Found ${#MODEL_PATHS[@]} model(s) to bench:"
+for p in "${MODEL_PATHS[@]}"; do
+  echo "  • $p"
+done
+echo
+
+declare -A CMDS=(
+  [rocm6_4_4]="toolbox run -c llama-rocm-6.4.4 -- /usr/local/bin/llama-bench"
+  [rocm6_4_4-rocwmma]="toolbox run -c llama-rocm-6.4.4-rocwmma -- /usr/local/bin/llama-bench"
+  [rocm7.1]="toolbox run -c llama-rocm-7.1 -- /usr/local/bin/llama-bench"
+  [rocm7.1-rocwmma]="toolbox run -c llama-rocm-7.1-rocwmma -- /usr/local/bin/llama-bench"
+  [rocm-7alpha-rocwmma-improved]="toolbox run -c llama-rocm-7alpha-rocwmma-improved -- /usr/local/bin/llama-bench"
+  [rocm-7alpha]="toolbox run -c llama-rocm-7alpha -- /usr/local/bin/llama-bench"
+  [rocm-7alpha-rocwmma]="toolbox run -c llama-rocm-7alpha-rocwmma -- /usr/local/bin/llama-bench"  
+  [rocm7_rc]="toolbox run -c llama-rocm-7rc -- /usr/local/bin/llama-bench"
+  [rocm7_rc-rocwmma]="toolbox run -c llama-rocm-7rc-rocwmma -- /usr/local/bin/llama-bench"
+  [vulkan_amdvlk]="toolbox run -c llama-vulkan-amdvlk -- /usr/sbin/llama-bench"
+  [vulkan_radv]="toolbox run -c llama-vulkan-radv -- /usr/sbin/llama-bench"
+)
+
+get_hblt_modes() {
+  local env="$1"
+  if [[ "$env" == rocm* ]]; then
+    printf '%s\n' default off
+  else
+    printf '%s\n' default
+  fi
+}
+
+for MODEL_PATH in "${MODEL_PATHS[@]}"; do
+  MODEL_NAME="$(basename "$MODEL_PATH" .gguf)"
+
+  for ENV in "${!CMDS[@]}"; do
+    CMD="${CMDS[$ENV]}"
+    mapfile -t HBLT_MODES < <(get_hblt_modes "$ENV")
+
+    for MODE in "${HBLT_MODES[@]}"; do
+      BASE_SUFFIX=""
+      CMD_EFFECTIVE="$CMD"
+
+      if [[ "$ENV" == rocm* ]]; then
+        if [[ "$MODE" == off ]]; then
+          BASE_SUFFIX="__hblt0"
+          CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=0 }"
+        else
+          CMD_EFFECTIVE="${CMD_EFFECTIVE/-- /-- env ROCBLAS_USE_HIPBLASLT=1 }"
+        fi
+      fi
+
+      # run twice: baseline and with flash attention
+      for FA in 1; do
+        SUFFIX="$BASE_SUFFIX"
+        EXTRA_ARGS=()
+        if (( FA == 1 )); then
+          SUFFIX="${SUFFIX}__fa1"
+          EXTRA_ARGS=( -fa 1 )
+        fi
+
+        for CTX in default longctx32768; do
+          CTX_SUFFIX=""
+          CTX_ARGS=()
+          if [[ "$CTX" == longctx32768 ]]; then
+            CTX_SUFFIX="__longctx32768"
+            CTX_ARGS=( -p 2048 -n 32 -d 32768 )
+            if [[ "$ENV" == *vulkan* ]]; then
+              CTX_ARGS+=( -ub 512 )
+            else
+              CTX_ARGS+=( -ub 2048 )
+            fi
+          fi
+
+          OUT="$RESULTDIR/${MODEL_NAME}__${ENV}${SUFFIX}${CTX_SUFFIX}.log"
+          CTX_REPS=3
+          if [[ "$CTX" == longctx32768 ]]; then
+            CTX_REPS=1
+          fi
+
+          if [[ -s "$OUT" ]]; then
+            echo "⏩ Skipping [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ ($CTX_SUFFIX)}, log already exists at $OUT"
+            continue
+          fi
+
+          FULL_CMD=( $CMD_EFFECTIVE -ngl 99 -mmp 0 -m "$MODEL_PATH" "${EXTRA_ARGS[@]}" "${CTX_ARGS[@]}" -r "$CTX_REPS" )
+
+          printf "\n▶ [%s] %s%s%s\n" "$ENV" "$MODEL_NAME" "${SUFFIX:+ $SUFFIX}" "${CTX_SUFFIX:+ $CTX_SUFFIX}"
+          printf "  → log: %s\n" "$OUT"
+          printf "  → cmd: %s\n\n" "${FULL_CMD[*]}"
+
+          if ! "${FULL_CMD[@]}" >"$OUT" 2>&1; then
+            status=$?
+            echo "✖ ! [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ $CTX_SUFFIX} failed (exit ${status})" >>"$OUT"
+            echo "  * [${ENV}] ${MODEL_NAME}${SUFFIX}${CTX_SUFFIX:+ $CTX_SUFFIX} : FAILED"
+          fi
+        done
+      done
+    done
+  done
+done