x

scripts/01-rocm-envs.sh

@@ -0,0 +1,4 @@
+export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
+export FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
+export VLLM_TARGET_DEVICE=rocm
+export VLLM_USE_TRITON_AWQ=1

scripts/99-toolbox-banner.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Lightweight banner with machine/GPU and ROCm version (vLLM edition)
+# No Triton env sourcing, same info/format as the image/video banner.
+
+# Only show for interactive shells
+case $- in *i*) ;; *) return 0 ;; esac
+
+oem_info() {
+  local v="" m="" d lv lm
+  for d in /sys/class/dmi/id /sys/devices/virtual/dmi/id; do
+    [[ -r "$d/sys_vendor" ]] && v=$(<"$d/sys_vendor")
+    [[ -r "$d/product_name" ]] && m=$(<"$d/product_name")
+    [[ -n "$v" || -n "$m" ]] && break
+  done
+  # ARM/SBC fallback
+  if [[ -z "$v" && -z "$m" && -r /proc/device-tree/model ]]; then
+    tr -d '\0' </proc/device-tree/model
+    return
+  fi
+  lv=$(printf '%s' "$v" | tr '[:upper:]' '[:lower:]')
+  lm=$(printf '%s' "$m" | tr '[:upper:]' '[:lower:]')
+  if [[ -n "$m" && "$lm" == "$lv "* ]]; then
+    printf '%s\n' "$m"
+  else
+    printf '%s %s\n' "${v:-Unknown}" "${m:-Unknown}"
+  fi
+}
+
+gpu_name() {
+  local name=""
+  if command -v rocm-smi >/dev/null 2>&1; then
+    name=$(rocm-smi --showproductname --csv 2>/dev/null | tail -n1 | cut -d, -f2)
+    [[ -z "$name" ]] && name=$(rocm-smi --showproductname 2>/dev/null | grep -m1 -E 'Product Name|Card series' | sed 's/.*: //')
+  fi
+  if [[ -z "$name" ]] && command -v rocminfo >/dev/null 2>&1; then
+    name=$(rocminfo 2>/dev/null | awk -F': ' '/^[[:space:]]*Name:/{print $2; exit}')
+  fi
+  if [[ -z "$name" ]] && command -v lspci >/dev/null 2>&1; then
+    name=$(lspci -nn 2>/dev/null | grep -Ei 'vga|display|gpu' | grep -i amd | head -n1 | cut -d: -f3-)
+  fi
+  # trim
+  name=$(printf '%s' "$name" | sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+$//' -e 's/[[:space:]]\{2,\}/ /g')
+  printf '%s\n' "${name:-Unknown AMD GPU}"
+}
+
+rocm_version() {
+  # Prefer the PyTorch HIP version from the venv, fallback to rocm pkg metadata
+  local PY="/torch-therock/.venv/bin/python"
+  [[ -x "$PY" ]] || PY="python"
+  "$PY" - <<'PY' 2>/dev/null || true
+try:
+    import torch
+    v = getattr(getattr(torch, "version", None), "hip", "") or ""
+    if v:
+        print(v)
+    else:
+        raise Exception("no torch.version.hip")
+except Exception:
+    try:
+        import importlib.metadata as im
+        try:
+            print(im.version("_rocm_sdk_core"))
+        except Exception:
+            print(im.version("rocm"))
+    except Exception:
+        print("")
+PY
+}
+
+MACHINE="$(oem_info)"
+GPU="$(gpu_name)"
+ROCM_VER="$(rocm_version)"
+
+echo
+cat <<'ASCII'
+ _____            _____  ______ ____  _   _    ___ ______ ___   ___  
+|  __ \     /\   |  __ \|  ____/ __ \| \ | |  / _ \____  / _ \ / _ \ 
+| |__) |   /  \  | |  | | |__ | |  | |  \| | | (_) |  / / | | | | | |
+|  _  /   / /\ \ | |  | |  __|| |  | | . ` |  \__, | / /| | | | | | |
+| | \ \  / ____ \| |__| | |___| |__| | |\  |    / / / / | |_| | |_| |
+|_|  \_\/_/    \_\_____/|______\____/|_| \_|   /_/ /_/   \___/ \___/ 
+                                                                     
+                         _____   _____  _____   ____  
+                   /\   |_   _| |  __ \|  __ \ / __ \ 
+                  /  \    | |   | |__) | |__) | |  | |
+                 / /\ \   | |   |  ___/|  _  /| |  | |
+                / ____ \ _| |_  | |    | | \ \| |__| |
+               /_/    \_\_____| |_|    |_|  \_\\____/ 
+
+                               v L L M                                      
+ASCII
+echo
+printf 'AMD R9700 — vLLM Toolbox (gfx1201, ROCm via TheRock)\n'
+[[ -n "$ROCM_VER" ]] && printf 'ROCm nightly: %s\n' "$ROCM_VER"
+echo
+printf 'Machine: %s\n' "$MACHINE"
+printf 'GPU    : %s\n\n' "$GPU"
+printf 'Repo   : https://github.com/kyuz0/amd-r9700-vllm-toolboxes\n'
+printf 'Image  : docker.io/kyuz0/vllm-therock-gfx1201:latest\n\n'
+printf 'Included:\n'
+printf '  - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling"
+printf '  - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct"
+printf '  - %-16s → %s\n' "API test"    "curl localhost:8000/v1/chat/completions"
+echo
+printf 'SSH tip: ssh -L 8000:localhost:8000 user@host\n\n'
+
+unset PROMPT_COMMAND
+PS1='\u@\h:\w\$ '

scripts/generate_models_list.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import sys
+import json
+import os
+from pathlib import Path
+
+# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path(__file__).parent.resolve()
+BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
+sys.path.append(str(BENCH_DIR))
+
+try:
+    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
+except ImportError:
+    # Fallback if run_vllm_bench not found
+    MODEL_TABLE = {}
+    MODELS_TO_RUN = []
+
+RESULTS_FILE = BENCH_DIR / "max_context_results.json"
+
+def get_best_context(model_id, max_tp):
+    """
+    Finds the maximum verified context for the given model
+    that fits within max_tp (system limit).
+    """
+    if not RESULTS_FILE.exists():
+        # Fallback to configured ctx in MODEL_TABLE
+        return int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192))
+        
+    try:
+        with open(RESULTS_FILE, "r") as f:
+            data = json.load(f)
+    except:
+        return 8192
+
+    best_ctx = 0
+    
+    # Filter for this model
+    candidates = [r for r in data if r["model"] == model_id and r["status"] == "success"]
+    
+    # Filter by TP <= max_tp (we can't launch TP2 on 1 GPU)
+    # But we WANT the limit for the Highest Allowable TP.
+    valid_candidates = [r for r in candidates if r["tp"] <= max_tp]
+    
+    if not valid_candidates:
+         # Fallback to hardcoded table
+         return int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192))
+
+    # Sort by Context Length (Descending) -> Then TP (Descending)
+    # This ensures we pick the biggest context possible on the hardware.
+    valid_candidates.sort(key=lambda x: (x["max_context_1_user"], x["tp"]), reverse=True)
+    
+    return valid_candidates[0]["max_context_1_user"]
+
+def main():
+    if len(sys.argv) > 1:
+        gpu_count = int(sys.argv[1])
+    else:
+        gpu_count = 1
+
+    for model_id in MODELS_TO_RUN:
+        config = MODEL_TABLE.get(model_id, {})
+        
+        # 1. Name: Use cleaner name
+        name = model_id.split("/")[-1]
+        
+        # 2. Repo: model_id
+        
+        # 3. MaxTP: Min of (Model valid tp max, System GPU Count)
+        valid_tps = config.get("valid_tp", [1])
+        model_max_tp = max(valid_tps) if valid_tps else 1
+        
+        # We cap the reported MaxTP at the system limit for the UI rangebox
+        # But for finding the context, we look at what is POSSIBLY supported.
+        # Actually, for the UI, we should only show what is switchable.
+        ui_max_tp = min(model_max_tp, gpu_count)
+        
+        if ui_max_tp < 1: ui_max_tp = 1 # Safety
+        
+        # 4. MaxCtx: Get from Results for this UI_MAX_TP
+        ctx = get_best_context(model_id, ui_max_tp)
+        
+        # 5. Flags
+        flags = []
+        if config.get("trust_remote"): flags.append("--trust-remote-code")
+        if config.get("enforce_eager"): flags.append("--enforce-eager")
+        flags_str = " ".join(flags)
+        
+        # 6. EnvVars
+        env_dict = config.get("env", {})
+        envs_str = " ".join([f"{k}={v}" for k,v in env_dict.items()])
+        
+        # Format: "Name|Repo|MaxCtx|MaxTP|Flags|EnvVars"
+        print(f"{name}|{model_id}|{ctx}|{ui_max_tp}|{flags_str}|{envs_str}")
+
+if __name__ == "__main__":
+    main()

scripts/start_vllm.py

@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import shutil
+import tempfile
+import subprocess
+from pathlib import Path
+
+# Add benchmarks dir to path to import config
+# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path(__file__).parent.resolve()
+BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
+OPT_DIR = Path("/opt")
+
+# Optional environment variable pointing to a local models directory.
+# If set, the script will prefer a subfolder under this path matching
+# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit)
+# when constructing the `vllm serve` command.
+LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
+
+
+# Check /opt first (Container), then local fallback
+if (OPT_DIR / "run_vllm_bench.py").exists():
+    sys.path.append(str(OPT_DIR))
+else:
+    sys.path.append(str(BENCH_DIR))
+
+try:
+    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
+except ImportError:
+    print("Error: Could not import run_vllm_bench.py config.")
+    sys.exit(1)
+
+if (OPT_DIR / "max_context_results.json").exists():
+    RESULTS_FILE = OPT_DIR / "max_context_results.json"
+else:
+    RESULTS_FILE = BENCH_DIR / "max_context_results.json"
+HOST = os.getenv("HOST", "0.0.0.0")
+PORT = os.getenv("PORT", "8000")
+
+def check_dependencies():
+    if not shutil.which("dialog"):
+        print("Error: 'dialog' is required. Please install it (apt-get install dialog).")
+        sys.exit(1)
+
+def detect_gpus():
+    """Detects AMD GPUs via rocm-smi or /dev/dri."""
+    try:
+        # Try rocm-smi first
+        res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if res.returncode == 0:
+            count = res.stdout.count("GPU")
+            if count > 0: return count
+    except: pass
+    
+    # Fallback to /dev/dri/render*
+    try:
+        return len(list(Path("/dev/dri").glob("renderD*")))
+    except:
+        return 1
+
+def get_verified_config(model_id, tp_size, max_seqs):
+    """
+    Reads max_context_results.json to find the best verified configuration.
+    Returns dict: {'ctx': int, 'util': float}
+    """
+    default_config = {
+        "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)),
+        "util": 0.90 # Safe default
+    }
+    
+    if not RESULTS_FILE.exists():
+        return default_config
+
+    try:
+        with open(RESULTS_FILE, "r") as f:
+            data = json.load(f)
+            
+        # Filter for Model + TP + Sequences
+        matches = [r for r in data 
+                  if r["model"] == model_id 
+                  and r["tp"] == tp_size 
+                  and r["max_seqs"] == max_seqs 
+                  and r["status"] == "success"]
+        
+        if not matches:
+            # Fallback 1: Try finding match with SAME TP but ANY Sequences (e.g. 1) to get base context?
+            # Actually, safer to fallback to default or try finding nearest sequence?
+            # Let's try finding exact match first. If fail, return default.
+            return default_config
+            
+        # Sort by Util desc, then Context desc
+        # We prefer higher utilization if available (performance), as long as it is verified success
+        matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True)
+        
+        best = matches[0]
+        return {
+            "ctx": best["max_context_1_user"],
+            "util": float(best["util"])
+        }
+        
+    except Exception as e:
+        return default_config
+
+def run_dialog(args):
+    """Runs dialog and returns stderr (selection)."""
+    with tempfile.NamedTemporaryFile(mode="w+") as tf:
+        cmd = ["dialog"] + args
+        try:
+            subprocess.run(cmd, stderr=tf, check=True)
+            tf.seek(0)
+            return tf.read().strip()
+        except subprocess.CalledProcessError:
+            return None # User cancelled
+
+def nuke_vllm_cache():
+    """Removes vLLM cache directory to fix potential graph/incompatibility issues."""
+    cache = Path.home() / ".cache" / "vllm"
+    if cache.exists():
+        try:
+            print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
+            subprocess.run(["rm", "-rf", str(cache)], check=True)
+            cache.mkdir(parents=True, exist_ok=True)
+            print(" Done.")
+            time.sleep(1)
+        except Exception as e:
+            print(f" Failed: {e}")
+
+def configure_and_launch(model_idx, gpu_count):
+    model_id = MODELS_TO_RUN[model_idx]
+    config = MODEL_TABLE[model_id]
+    
+    # Determine whether we have a local copy to serve. Try multiple fallbacks:
+    # 1) LOCAL_MODEL_DIR/<owner>/<repo>
+    # 2) LOCAL_MODEL_DIR/<repo>
+    # 3) case-insensitive match of <repo> in LOCAL_MODEL_DIR
+    model_path = model_id
+    if LOCAL_MODEL_DIR:
+        # Full repo path (owner/repo)
+        candidate_full = os.path.join(LOCAL_MODEL_DIR, model_id)
+        if os.path.isdir(candidate_full):
+            model_path = candidate_full
+        else:
+            # Repo-name only (last segment)
+            repo_name = model_id.split('/')[-1]
+            candidate_repo = os.path.join(LOCAL_MODEL_DIR, repo_name)
+            if os.path.isdir(candidate_repo):
+                model_path = candidate_repo
+            else:
+                # Fallback: try to find a directory in LOCAL_MODEL_DIR that matches repo_name case-insensitively
+                try:
+                    for entry in os.listdir(LOCAL_MODEL_DIR):
+                        if entry.lower() == repo_name.lower():
+                            entry_path = os.path.join(LOCAL_MODEL_DIR, entry)
+                            if os.path.isdir(entry_path):
+                                model_path = entry_path
+                                break
+                except Exception:
+                    pass
++    # if LOCAL_MODEL_DIR is specified, refuse to fall back to remote
++    if LOCAL_MODEL_DIR and model_path == model_id:
++        print(f"Error: model '{model_id}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}")
++        print("Off‑line mode active; network downloads are disabled.")
++        sys.exit(1)
+    
+    # Static Config
+    valid_tps = config.get("valid_tp", [1])
+    max_tp = max(valid_tps) if valid_tps else 1
+    
+    # Defaults
+    current_tp = min(gpu_count, max_tp)
+    current_seqs = 1 # Default to 1 concurrent user/request for stability
+    
+    # Initial Lookup
+    verified = get_verified_config(model_id, current_tp, current_seqs)
+    current_ctx = verified["ctx"]
+    current_util = verified["util"]
+    
+    clear_cache = False
+    use_eager = config.get("enforce_eager", False) # Default to model config, usually False
+    use_rocm_attn = False # Default to Triton
+    
+    name = model_id.split("/")[-1]
+    
+    while True:
+        cache_status = "YES" if clear_cache else "NO"
+        eager_status = "YES" if use_eager else "NO"
+        attn_backend = "ROCm" if use_rocm_attn else "Triton"
+        
+        menu_args = [
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", f"Configuration: {name}",
+            "--menu", "Customize Launch Parameters:", "22", "65", "9",
+            "1", f"Tensor Parallelism:   {current_tp}",
+            "2", f"Concurrent Requests:  {current_seqs}",
+            "3", f"Context Length:       {current_ctx} (Verified)",
+            "4", f"GPU Utilization:      {current_util} (Verified)",
+            "5", f"Attention Backend:    {attn_backend}",
+            "6", f"Erase vLLM Cache:     {cache_status}",
+            "7", f"Force Eager Mode:     {eager_status}",
+            "8", "LAUNCH SERVER"
+        ]
+        
+        choice = run_dialog(menu_args)
+        if not choice: return False # Back/Cancel
+        
+        if choice == "1":
+            # TP Selection
+            new_tp = run_dialog([
+                "--title", "Tensor Parallelism",
+                "--rangebox", f"Set TP Size (1-{max_tp})", "10", "40", "1", str(max_tp), str(current_tp)
+            ])
+            if new_tp: 
+                new_tp_int = int(new_tp)
+                if new_tp_int != current_tp:
+                    current_tp = new_tp_int
+                    # RE-CALCULATE Config
+                    verified = get_verified_config(model_id, current_tp, current_seqs)
+                    current_ctx = verified["ctx"]
+                    current_util = verified["util"]
+            
+        elif choice == "2":
+            # Max Seqs Selection
+            new_seqs = run_dialog([
+                "--title", "Concurrent Requests",
+                "--menu", "Select Max Concurrent Requests:", "12", "40", "4",
+                "1", "1 (Latency Focus)",
+                "4", "4 (Balanced)",
+                "8", "8 (Throughput)",
+                "16", "16 (Max Load)"
+            ])
+            if new_seqs:
+                current_seqs = int(new_seqs)
+                # RE-CALCULATE Config based on new concurrency
+                verified = get_verified_config(model_id, current_tp, current_seqs)
+                current_ctx = verified["ctx"]
+                current_util = verified["util"]
+
+        elif choice == "3":
+            # Configured Length Override
+            new_ctx = run_dialog([
+                "--title", "Context Length",
+                "--inputbox", f"Override verified limit ({current_ctx}):", "10", "40", str(current_ctx)
+            ])
+            if new_ctx: current_ctx = int(new_ctx)
+
+        elif choice == "4":
+             # Util Override
+             pass 
+
+        elif choice == "5":
+            # Toggle Attention Backend
+            use_rocm_attn = not use_rocm_attn
+
+        elif choice == "6":
+            # Toggle Cache
+            if not clear_cache:
+                # Enabling it -> Show Warning
+                warn_msg = (
+                    "WARNING: Erasing the vLLM cache will remove the compiled compute graphs.\n\n"
+                    "This is useful if you are experiencing crashes, 'invalid graph' errors,\n"
+                    "or have switched vLLM versions recently.\n\n"
+                    "However, the next startup will take longer as graphs are re-compiled.\n\n"
+                    "Are you sure you want to enable this?"
+                )
+                confirm = run_dialog([
+                    "--title", "Erase Cache Warning", 
+                    "--yesno", warn_msg, "12", "60"
+                ])
+                
+                # If confirm is not None (exit 0), it is YES.
+                if confirm is not None:
+                     clear_cache = True
+            else:
+                # Disabling it -> No warning needed
+                clear_cache = False
+             
+        elif choice == "7":
+            # Toggle Eager Mode
+            use_eager = not use_eager
+             
+        elif choice == "8":
+            # Launch
+            break
+            
+    # Build Command
+    subprocess.run(["clear"])
+    
+    if clear_cache:
+        nuke_vllm_cache()
+    
+    cmd = [
+        "vllm", "serve", model_path,
+        "--host", HOST,
+        "--port", PORT,
+        "--tensor-parallel-size", str(current_tp),
+        "--max-num-seqs", str(current_seqs),
+        "--max-model-len", str(current_ctx),
+        "--gpu-memory-utilization", str(current_util),
+        "--dtype", "auto"
+    ]
+    
+    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if use_eager: cmd.append("--enforce-eager")
+    
+    # Env Vars
+    env = os.environ.copy()
+    env.update(config.get("env", {}))
+    
+    if use_rocm_attn:
+        env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
+        env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
+        # Optional: Explicitly mention these in print
+        
+    
+    print("\n" + "="*60)
+    print(f" Launching: {name}")
+    if model_path != model_id:
+        print(f" (using local model at {model_path})")
+    print(f" Config:    TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx} | Util={current_util}")
+    print(f" Backend:   {'ROCm' if use_rocm_attn else 'Triton'}")
+    if clear_cache:
+        print(f" Action:    Clearing vLLM Cache (~/.cache/vllm)")
+    print(f" Command:   {' '.join(cmd)}")
+    print("="*60 + "\n")
+    
+    os.execvpe("vllm", cmd, env)
+
+def main():
+    check_dependencies()
+    gpu_count = detect_gpus()
+    
+    while True:
+        # Build Model Menu
+        menu_items = []
+        for i, m_id in enumerate(MODELS_TO_RUN):
+            name = m_id.split("/")[-1]
+            # Pre-calc verified ctx for 'default' TP to show in menu? 
+            # Or just show names. Just names is cleaner.
+            config = MODEL_TABLE[m_id]
+            menu_items.extend([str(i), name])
+            
+        choice = run_dialog([
+            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+            "--title", "Select Model",
+            "--menu", "Choose a model to serve:", "20", "60", "10"
+        ] + menu_items)
+        
+        if not choice:
+            subprocess.run(["clear"])
+            print("Selection cancelled.")
+            sys.exit(0)
+            
+        configure_and_launch(int(choice), gpu_count)
+
+if __name__ == "__main__":
+    main()

scripts/zz-venv-last.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Ensure /opt/venv/bin is first even if ~/.local/bin or ~/.cargo/bin prepend themselves via user dotfiles.
+
+_venv_path_fix() {
+  # remove any existing /opt/venv/bin entries, then prepend one
+  local newpath
+  newpath="$(printf '%s' "$PATH" | awk -v RS=: -v ORS=: '$0!="/opt/venv/bin"{print}')"
+  PATH="/opt/venv/bin:${newpath%:}"
+}
+
+# run once after shell init; don't duplicate
+case "$PROMPT_COMMAND" in
+  *_venv_path_fix*) : ;;
+  *) PROMPT_COMMAND="_venv_path_fix${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;;
+esac
+