cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
SZCJW
2026-03-26 21:37:10 +08:00
parent 81ffbf61c0
commit 309e6fbf0d
4 changed files with 132 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+4 -1
View File
@@ -161,4 +161,7 @@ COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py
RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json
RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh
CMD ["/bin/bash"]
# Set environment variable for default model usage
ENV USE_DEFAULT_MODEL=true
CMD ["start-vllm"]
config.json
+57
View File
@@ -0,0 +1,57 @@
{
"default_model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
"models": {
"meta-llama/Meta-Llama-3.1-8B-Instruct": {
"ctx": "65536",
"trust_remote": false,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
},
"openai/gpt-oss-20b": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "8192"
},
"RedHatAI/Qwen3-14B-FP8-dynamic": {
"ctx": "32768",
"trust_remote": true,
"valid_tp": [1],
"max_num_seqs": "64",
"max_tokens": "32768"
},
"cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": {
"ctx": "24576",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "32768"
},
"cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": {
"ctx": "20480",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "16384",
"enforce_eager": false,
"env": {"VLLM_USE_TRITON_AWQ": "1"}
},
"RedHatAI/gemma-3-27b-it-FP8-dynamic": {
"ctx": "29000",
"trust_remote": true,
"valid_tp": [2],
"max_num_seqs": "32",
"max_tokens": "29000",
"gpu_util": "0.94"
},
"RedHatAI/gemma-3-12b-it-FP8-dynamic": {
"ctx": "9900",
"trust_remote": true,
"valid_tp": [1, 2],
"max_num_seqs": "64",
"max_tokens": "9900"
}
}
}
docker-compose.yml
+23
View File
@@ -0,0 +1,23 @@
version: '3.8'
services:
vllm:
build: .
container_name: vllm-r9700
environment:
- USE_DEFAULT_MODEL=true
- LOCAL_MODEL_DIR=/opt/model
volumes:
- /opt/project/amd-r9700-vllm-toolboxes/config.json:/config.json:ro
- /opt/model:/opt/model
devices:
- /dev/dri:/dev/dri
- /dev/kfd:/dev/kfd
group_add:
- video
- render
security_opt:
- seccomp=unconfined
ports:
- "8000:8000"
restart: unless-stopped
scripts/start_vllm.py
+48 -36
View File
@@ -7,29 +7,28 @@ import tempfile
import subprocess
from pathlib import Path
# Add benchmarks dir to path to import config
# Add benchmarks dir to path to import config
SCRIPT_DIR = Path(__file__).parent.resolve()
BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
OPT_DIR = Path("/opt")
# Optional environment variable pointing to a local models directory.
# If set, the script will prefer a subfolder under this path matching
# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit)
# when constructing the `vllm serve` command.
LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
# Config file path (check container path first, then local path)
CONFIG_PATH = Path("/config.json")
if not CONFIG_PATH.exists():
CONFIG_PATH = SCRIPT_DIR.parent / "config.json"
# Local model directory (container path)
LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", str(SCRIPT_DIR.parent / "models"))
# Check /opt first (Container), then local fallback
if (OPT_DIR / "run_vllm_bench.py").exists():
sys.path.append(str(OPT_DIR))
else:
sys.path.append(str(BENCH_DIR))
# Load configuration from config.json
try:
from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
except ImportError:
print("Error: Could not import run_vllm_bench.py config.")
with open(CONFIG_PATH, "r") as f:
config_data = json.load(f)
MODEL_TABLE = config_data["models"]
DEFAULT_MODEL = config_data["default_model"]
MODELS_TO_RUN = list(MODEL_TABLE.keys())
except Exception as e:
print(f"Error: Could not load config.json: {e}")
sys.exit(1)
if (OPT_DIR / "max_context_results.json").exists():
@@ -331,28 +330,41 @@ def main():
check_dependencies()
gpu_count = detect_gpus()
while True:
# Build Model Menu
menu_items = []
for i, m_id in enumerate(MODELS_TO_RUN):
name = m_id.split("/")[-1]
# Pre-calc verified ctx for 'default' TP to show in menu?
# Or just show names. Just names is cleaner.
config = MODEL_TABLE[m_id]
menu_items.extend([str(i), name])
# Check if we should use default model (for docker startup)
use_default = os.getenv("USE_DEFAULT_MODEL", "false").lower() == "true"
if use_default:
# Find the index of default model
try:
default_idx = MODELS_TO_RUN.index(DEFAULT_MODEL)
print(f"Using default model: {DEFAULT_MODEL}")
configure_and_launch(default_idx, gpu_count)
except ValueError:
print(f"Error: Default model {DEFAULT_MODEL} not found in configuration")
sys.exit(1)
else:
while True:
# Build Model Menu
menu_items = []
for i, m_id in enumerate(MODELS_TO_RUN):
name = m_id.split("/")[-1]
# Mark default model
if m_id == DEFAULT_MODEL:
name += " (Default)"
menu_items.extend([str(i), name])
choice = run_dialog([
"--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
"--title", "Select Model",
"--menu", "Choose a model to serve:", "20", "60", "10"
] + menu_items)
choice = run_dialog([
"--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
"--title", "Select Model",
"--menu", "Choose a model to serve:", "20", "60", "10"
] + menu_items)
if not choice:
subprocess.run(["clear"])
print("Selection cancelled.")
sys.exit(0)
configure_and_launch(int(choice), gpu_count)
if not choice:
subprocess.run(["clear"])
print("Selection cancelled.")
sys.exit(0)
configure_and_launch(int(choice), gpu_count)
if __name__ == "__main__":
main()