init

2026-03-10 23:09:05 +08:00
parent 1c08c96d32
commit 7a38199dde
16 changed files with 1702 additions and 1 deletions
@@ -0,0 +1,4 @@
+export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
+export FLASH_ATTENTION_TRITON_AMD_ENABLE="TRUE"
+export VLLM_TARGET_DEVICE=rocm
+export VLLM_USE_TRITON_AWQ=1
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# Lightweight banner with machine/GPU and ROCm version (vLLM edition)
+# No Triton env sourcing, same info/format as the image/video banner.
+
+# Only show for interactive shells
+case $- in *i*) ;; *) return 0 ;; esac
+
+oem_info() {
+  local v="" m="" d lv lm
+  for d in /sys/class/dmi/id /sys/devices/virtual/dmi/id; do
+    [[ -r "$d/sys_vendor" ]] && v=$(<"$d/sys_vendor")
+    [[ -r "$d/product_name" ]] && m=$(<"$d/product_name")
+    [[ -n "$v" || -n "$m" ]] && break
+  done
+  # ARM/SBC fallback
+  if [[ -z "$v" && -z "$m" && -r /proc/device-tree/model ]]; then
+    tr -d '\0' </proc/device-tree/model
+    return
+  fi
+  lv=$(printf '%s' "$v" | tr '[:upper:]' '[:lower:]')
+  lm=$(printf '%s' "$m" | tr '[:upper:]' '[:lower:]')
+  if [[ -n "$m" && "$lm" == "$lv "* ]]; then
+    printf '%s\n' "$m"
+  else
+    printf '%s %s\n' "${v:-Unknown}" "${m:-Unknown}"
+  fi
+}
+
+gpu_name() {
+  local name=""
+  if command -v rocm-smi >/dev/null 2>&1; then
+    name=$(rocm-smi --showproductname --csv 2>/dev/null | tail -n1 | cut -d, -f2)
+    [[ -z "$name" ]] && name=$(rocm-smi --showproductname 2>/dev/null | grep -m1 -E 'Product Name|Card series' | sed 's/.*: //')
+  fi
+  if [[ -z "$name" ]] && command -v rocminfo >/dev/null 2>&1; then
+    name=$(rocminfo 2>/dev/null | awk -F': ' '/^[[:space:]]*Name:/{print $2; exit}')
+  fi
+  if [[ -z "$name" ]] && command -v lspci >/dev/null 2>&1; then
+    name=$(lspci -nn 2>/dev/null | grep -Ei 'vga|display|gpu' | grep -i amd | head -n1 | cut -d: -f3-)
+  fi
+  # trim
+  name=$(printf '%s' "$name" | sed -e 's/^[[:space:]]\+//' -e 's/[[:space:]]\+$//' -e 's/[[:space:]]\{2,\}/ /g')
+  printf '%s\n' "${name:-Unknown AMD GPU}"
+}
+
+rocm_version() {
+  # Prefer the PyTorch HIP version from the venv, fallback to rocm pkg metadata
+  local PY="/torch-therock/.venv/bin/python"
+  [[ -x "$PY" ]] || PY="python"
+  "$PY" - <<'PY' 2>/dev/null || true
+try:
+    import torch
+    v = getattr(getattr(torch, "version", None), "hip", "") or ""
+    if v:
+        print(v)
+    else:
+        raise Exception("no torch.version.hip")
+except Exception:
+    try:
+        import importlib.metadata as im
+        try:
+            print(im.version("_rocm_sdk_core"))
+        except Exception:
+            print(im.version("rocm"))
+    except Exception:
+        print("")
+PY
+}
+
+MACHINE="$(oem_info)"
+GPU="$(gpu_name)"
+ROCM_VER="$(rocm_version)"
+
+echo
+cat <<'ASCII'
+ _____            _____  ______ ____  _   _    ___ ______ ___   ___  
+|  __ \     /\   |  __ \|  ____/ __ \| \ | |  / _ \____  / _ \ / _ \ 
+| |__) |   /  \  | |  | | |__ | |  | |  \| | | (_) |  / / | | | | | |
+|  _  /   / /\ \ | |  | |  __|| |  | | . ` |  \__, | / /| | | | | | |
+| | \ \  / ____ \| |__| | |___| |__| | |\  |    / / / / | |_| | |_| |
+|_|  \_\/_/    \_\_____/|______\____/|_| \_|   /_/ /_/   \___/ \___/ 
+                                                                     
+                         _____   _____  _____   ____  
+                   /\   |_   _| |  __ \|  __ \ / __ \ 
+                  /  \    | |   | |__) | |__) | |  | |
+                 / /\ \   | |   |  ___/|  _  /| |  | |
+                / ____ \ _| |_  | |    | | \ \| |__| |
+               /_/    \_\_____| |_|    |_|  \_\\____/ 
+
+                               v L L M                                      
+ASCII
+echo
+printf 'AMD R9700 — vLLM Toolbox (gfx1201, ROCm via TheRock)\n'
+[[ -n "$ROCM_VER" ]] && printf 'ROCm nightly: %s\n' "$ROCM_VER"
+echo
+printf 'Machine: %s\n' "$MACHINE"
+printf 'GPU    : %s\n\n' "$GPU"
+printf 'Repo   : https://github.com/kyuz0/amd-r9700-vllm-toolboxes\n'
+printf 'Image  : docker.io/kyuz0/vllm-therock-gfx1201:latest\n\n'
+printf 'Included:\n'
+printf '  - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling"
+printf '  - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct"
+printf '  - %-16s → %s\n' "API test"    "curl localhost:8000/v1/chat/completions"
+echo
+printf 'SSH tip: ssh -L 8000:localhost:8000 user@host\n\n'
+
+unset PROMPT_COMMAND
+PS1='\u@\h:\w\$ '
@@ -0,0 +1,235 @@
+#!/usr/bin/env python3
+import sys
+import os
+import json
+import yaml
+import shutil
+import subprocess
+from pathlib import Path
+
+# Add benchmarks dir to path to import config
+SCRIPT_DIR = Path(__file__).parent.resolve()
+OPT_DIR = Path("/opt")
+
+# Required environment variable pointing to a local models directory
+LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
+if not LOCAL_MODEL_DIR:
+    print("Error: LOCAL_MODEL_DIR environment variable is required.")
+    sys.exit(1)
+
+# Configuration file path
+CONFIG_FILE = os.getenv("VLLM_CONFIG_FILE", "/etc/vllm/model_config.yaml")
+
+# Default configuration
+DEFAULT_CONFIG = {
+    "default": "",
+    "models": {},
+    "server": {
+        "host": "0.0.0.0",
+        "log_level": "info"
+    }
+}
+
+def detect_gpus():
+    """Detects AMD GPUs via rocm-smi or /dev/dri."""
+    try:
+        # Try rocm-smi first
+        res = subprocess.run(["rocm-smi", "--showid", "--csv"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        if res.returncode == 0:
+            count = res.stdout.count("GPU")
+            if count > 0: return count
+    except: pass
+    
+    # Fallback to /dev/dri/render*
+    try:
+        return len(list(Path("/dev/dri").glob("renderD*")))
+    except:
+        return 1
+
+def load_config():
+    """Load configuration from YAML file."""
+    config = DEFAULT_CONFIG.copy()
+    
+    if Path(CONFIG_FILE).exists():
+        try:
+            with open(CONFIG_FILE, "r", encoding="utf-8") as f:
+                user_config = yaml.safe_load(f)
+                if user_config:
+                    config.update(user_config)
+        except Exception as e:
+            print(f"Warning: Failed to load config file: {e}")
+            print("Using default configuration.")
+    else:
+        print(f"Warning: Config file not found at {CONFIG_FILE}")
+        print("Using default configuration.")
+    
+    return config
+
+def nuke_vllm_cache():
+    """Removes vLLM cache directory to fix potential graph/incompatibility issues."""
+    cache = Path.home() / ".cache" / "vllm"
+    if cache.exists():
+        try:
+            print(f"Clearing vLLM cache at {cache}...", end="", flush=True)
+            subprocess.run(["rm", "-rf", str(cache)], check=True)
+            cache.mkdir(parents=True, exist_ok=True)
+            print(" Done.")
+        except Exception as e:
+            print(f" Failed: {e}")
+
+def get_local_model_path(model_name):
+    """Get local model path from LOCAL_MODEL_DIR."""
+    # Try multiple paths:
+    # 1) LOCAL_MODEL_DIR/model_name
+    # 2) case-insensitive match in LOCAL_MODEL_DIR
+    
+    # Exact match
+    candidate_exact = os.path.join(LOCAL_MODEL_DIR, model_name)
+    if os.path.isdir(candidate_exact):
+        return candidate_exact
+    
+    # Case-insensitive match
+    try:
+        for entry in os.listdir(LOCAL_MODEL_DIR):
+            if entry.lower() == model_name.lower():
+                entry_path = os.path.join(LOCAL_MODEL_DIR, entry)
+                if os.path.isdir(entry_path):
+                    return entry_path
+    except Exception as e:
+        print(f"Error searching for model: {e}")
+    
+    return None
+
+def verify_model_path(model_path):
+    """Verify that the model path contains required files."""
+    required = ["config.json", "pytorch_model.bin", "model.safetensors"]
+    found = any(os.path.isfile(os.path.join(model_path, f)) for f in required)
+    if not found:
+        print(f"Error: local model dir {model_path} missing expected files {required}")
+        return False
+    return True
+
+def main():
+    # Load configuration
+    config = load_config()
+    
+    # Get model name from command line or use default
+    model_name = None
+    if len(sys.argv) > 1:
+        model_name = sys.argv[1]
+    
+    if not model_name:
+        model_name = config.get("default", "")
+        if not model_name:
+            print("Error: Either specify a model name as argument or set 'default' in config.")
+            sys.exit(1)
+    
+    # Get model configuration
+    model_config = config.get("models", {}).get(model_name)
+    if not model_config:
+        print(f"Error: Model '{model_name}' not found in configuration.")
+        sys.exit(1)
+    
+    # Get model path
+    model_path = model_config.get("path")
+    if not model_path:
+        # Fallback to LOCAL_MODEL_DIR if path not specified
+        model_path = get_local_model_path(model_name)
+        if not model_path:
+            print(f"Error: model '{model_name}' not found under LOCAL_MODEL_DIR={LOCAL_MODEL_DIR}")
+            sys.exit(1)
+    
+    # Verify model path
+    if not verify_model_path(model_path):
+        sys.exit(1)
+    
+    # Detect GPU count
+    gpu_count = detect_gpus()
+    
+    # Get server configuration
+    server_config = config.get("server", {})
+    host = server_config.get("host", "0.0.0.0")
+    
+    # Build command
+    cmd = ["vllm", "serve", model_path]
+    
+    # Add server parameters
+    cmd.extend(["--host", host])
+    if "port" in model_config:
+        cmd.extend(["--port", str(model_config["port"])])
+    
+    # Add API key for OpenAI compatibility
+    api_key = model_config.get("api_key")
+    if api_key:
+        cmd.extend(["--api-key", api_key])
+    
+    # Add model parameters
+    if "tensor_parallel_size" in model_config:
+        tp_size = min(model_config["tensor_parallel_size"], gpu_count)
+        cmd.extend(["--tensor-parallel-size", str(tp_size)])
+    
+    if "max_num_seqs" in model_config:
+        cmd.extend(["--max-num-seqs", str(model_config["max_num_seqs"])])
+    
+    if "max_model_len" in model_config:
+        cmd.extend(["--max-model-len", str(model_config["max_model_len"])])
+    
+    if "gpu_memory_utilization" in model_config:
+        cmd.extend(["--gpu-memory-utilization", str(model_config["gpu_memory_utilization"])])
+    
+    if "dtype" in model_config:
+        cmd.extend(["--dtype", model_config["dtype"]])
+    else:
+        cmd.extend(["--dtype", "auto"])
+    
+    if "max_num_batched_tokens" in model_config:
+        cmd.extend(["--max-num-batched-tokens", str(model_config["max_num_batched_tokens"])])
+    
+    if "block_size" in model_config:
+        cmd.extend(["--block-size", str(model_config["block_size"])])
+    
+    if "swap_space" in model_config:
+        cmd.extend(["--swap-space", str(model_config["swap_space"])])
+    
+    if model_config.get("enforce_eager", False):
+        cmd.append("--enforce-eager")
+    
+    if model_config.get("trust_remote", False):
+        cmd.append("--trust-remote-code")
+    
+    # Set environment variables
+    env = os.environ.copy()
+    
+    if model_config.get("use_rocm_attn", False):
+        env["VLLM_V1_USE_PREFILL_DECODE_ATTENTION"] = "1"
+        env["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
+    
+    # Clear cache if requested
+    if model_config.get("clear_cache", False):
+        nuke_vllm_cache()
+    
+    # Print configuration
+    print("\n" + "="*60)
+    print(f" Launching: {model_name}")
+    print(f" Model Path: {model_path}")
+    print(f" Host:      {host}")
+    if "port" in model_config:
+        print(f" Port:      {model_config['port']}")
+    if api_key:
+        print(f" API Key:   {api_key[:8]}...{api_key[-4:]}")
+    if "tensor_parallel_size" in model_config:
+        print(f" TP Size:   {tp_size}")
+    if "max_num_seqs" in model_config:
+        print(f" Max Seqs:  {model_config['max_num_seqs']}")
+    if "max_model_len" in model_config:
+        print(f" Max Ctx:   {model_config['max_model_len']}")
+    if "gpu_memory_utilization" in model_config:
+        print(f" GPU Util:  {model_config['gpu_memory_utilization']}")
+    print(f" Command:   {' '.join(cmd)}")
+    print("="*60 + "\n")
+    
+    # Launch vLLM server
+    os.execvpe("vllm", cmd, env)
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Ensure /opt/venv/bin is first even if ~/.local/bin or ~/.cargo/bin prepend themselves via user dotfiles.
+
+_venv_path_fix() {
+  # remove any existing /opt/venv/bin entries, then prepend one
+  local newpath
+  newpath="$(printf '%s' "$PATH" | awk -v RS=: -v ORS=: '$0!="/opt/venv/bin"{print}')"
+  PATH="/opt/venv/bin:${newpath%:}"
+}
+
+# run once after shell init; don't duplicate
+case "$PROMPT_COMMAND" in
+  *_venv_path_fix*) : ;;
+  *) PROMPT_COMMAND="_venv_path_fix${PROMPT_COMMAND:+;$PROMPT_COMMAND}" ;;
+esac
+