diff --git a/Dockerfile b/Dockerfile index 8853520..aa10035 100644 --- a/Dockerfile +++ b/Dockerfile @@ -161,4 +161,7 @@ COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh -CMD ["/bin/bash"] +# Set environment variable for default model usage +ENV USE_DEFAULT_MODEL=true + +CMD ["start-vllm"] diff --git a/config.json b/config.json new file mode 100644 index 0000000..d52fe3c --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "default_model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "models": { + "meta-llama/Meta-Llama-3.1-8B-Instruct": { + "ctx": "65536", + "trust_remote": false, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + "openai/gpt-oss-20b": { + "ctx": "32768", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "8192" + }, + "RedHatAI/Qwen3-14B-FP8-dynamic": { + "ctx": "32768", + "trust_remote": true, + "valid_tp": [1], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit": { + "ctx": "24576", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "32768" + }, + "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { + "ctx": "20480", + "trust_remote": true, + "valid_tp": [2], + "max_num_seqs": "32", + "max_tokens": "16384", + "enforce_eager": false, + "env": {"VLLM_USE_TRITON_AWQ": "1"} + }, + "RedHatAI/gemma-3-27b-it-FP8-dynamic": { + "ctx": "29000", + "trust_remote": true, + "valid_tp": [2], + "max_num_seqs": "32", + "max_tokens": "29000", + "gpu_util": "0.94" + }, + "RedHatAI/gemma-3-12b-it-FP8-dynamic": { + "ctx": "9900", + "trust_remote": true, + "valid_tp": [1, 2], + "max_num_seqs": "64", + "max_tokens": "9900" + } + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2322763 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3.8' + +services: + vllm: + build: . + container_name: vllm-r9700 + environment: + - USE_DEFAULT_MODEL=true + - LOCAL_MODEL_DIR=/opt/model + volumes: + - /opt/project/amd-r9700-vllm-toolboxes/config.json:/config.json:ro + - /opt/model:/opt/model + devices: + - /dev/dri:/dev/dri + - /dev/kfd:/dev/kfd + group_add: + - video + - render + security_opt: + - seccomp=unconfined + ports: + - "8000:8000" + restart: unless-stopped \ No newline at end of file diff --git a/scripts/start_vllm.py b/scripts/start_vllm.py index a33cd1e..717b761 100644 --- a/scripts/start_vllm.py +++ b/scripts/start_vllm.py @@ -7,29 +7,28 @@ import tempfile import subprocess from pathlib import Path -# Add benchmarks dir to path to import config # Add benchmarks dir to path to import config SCRIPT_DIR = Path(__file__).parent.resolve() BENCH_DIR = SCRIPT_DIR.parent / "benchmarks" OPT_DIR = Path("/opt") -# Optional environment variable pointing to a local models directory. -# If set, the script will prefer a subfolder under this path matching -# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit) -# when constructing the `vllm serve` command. -LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR") +# Config file path (check container path first, then local path) +CONFIG_PATH = Path("/config.json") +if not CONFIG_PATH.exists(): + CONFIG_PATH = SCRIPT_DIR.parent / "config.json" +# Local model directory (container path) +LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", str(SCRIPT_DIR.parent / "models")) -# Check /opt first (Container), then local fallback -if (OPT_DIR / "run_vllm_bench.py").exists(): - sys.path.append(str(OPT_DIR)) -else: - sys.path.append(str(BENCH_DIR)) - +# Load configuration from config.json try: - from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN -except ImportError: - print("Error: Could not import run_vllm_bench.py config.") + with open(CONFIG_PATH, "r") as f: + config_data = json.load(f) + MODEL_TABLE = config_data["models"] + DEFAULT_MODEL = config_data["default_model"] + MODELS_TO_RUN = list(MODEL_TABLE.keys()) +except Exception as e: + print(f"Error: Could not load config.json: {e}") sys.exit(1) if (OPT_DIR / "max_context_results.json").exists(): @@ -331,28 +330,41 @@ def main(): check_dependencies() gpu_count = detect_gpus() - while True: - # Build Model Menu - menu_items = [] - for i, m_id in enumerate(MODELS_TO_RUN): - name = m_id.split("/")[-1] - # Pre-calc verified ctx for 'default' TP to show in menu? - # Or just show names. Just names is cleaner. - config = MODEL_TABLE[m_id] - menu_items.extend([str(i), name]) + # Check if we should use default model (for docker startup) + use_default = os.getenv("USE_DEFAULT_MODEL", "false").lower() == "true" + + if use_default: + # Find the index of default model + try: + default_idx = MODELS_TO_RUN.index(DEFAULT_MODEL) + print(f"Using default model: {DEFAULT_MODEL}") + configure_and_launch(default_idx, gpu_count) + except ValueError: + print(f"Error: Default model {DEFAULT_MODEL} not found in configuration") + sys.exit(1) + else: + while True: + # Build Model Menu + menu_items = [] + for i, m_id in enumerate(MODELS_TO_RUN): + name = m_id.split("/")[-1] + # Mark default model + if m_id == DEFAULT_MODEL: + name += " (Default)" + menu_items.extend([str(i), name]) + + choice = run_dialog([ + "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})", + "--title", "Select Model", + "--menu", "Choose a model to serve:", "20", "60", "10" + ] + menu_items) - choice = run_dialog([ - "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})", - "--title", "Select Model", - "--menu", "Choose a model to serve:", "20", "60", "10" - ] + menu_items) - - if not choice: - subprocess.run(["clear"]) - print("Selection cancelled.") - sys.exit(0) - - configure_and_launch(int(choice), gpu_count) + if not choice: + subprocess.run(["clear"]) + print("Selection cancelled.") + sys.exit(0) + + configure_and_launch(int(choice), gpu_count) if __name__ == "__main__": main()