init

scripts/start_vllm.py

@@ -7,29 +7,28 @@ import tempfile
 import subprocess
 from pathlib import Path
 
-# Add benchmarks dir to path to import config
 # Add benchmarks dir to path to import config
 SCRIPT_DIR = Path(__file__).parent.resolve()
 BENCH_DIR = SCRIPT_DIR.parent / "benchmarks"
 OPT_DIR = Path("/opt")
 
-# Optional environment variable pointing to a local models directory.
-# If set, the script will prefer a subfolder under this path matching
-# the model repo ID (e.g. LOCAL_MODEL_DIR/cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit)
-# when constructing the `vllm serve` command.
-LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR")
+# Config file path (check container path first, then local path)
+CONFIG_PATH = Path("/config.json")
+if not CONFIG_PATH.exists():
+    CONFIG_PATH = SCRIPT_DIR.parent / "config.json"
 
+# Local model directory (container path)
+LOCAL_MODEL_DIR = os.getenv("LOCAL_MODEL_DIR", str(SCRIPT_DIR.parent / "models"))
 
-# Check /opt first (Container), then local fallback
-if (OPT_DIR / "run_vllm_bench.py").exists():
-    sys.path.append(str(OPT_DIR))
-else:
-    sys.path.append(str(BENCH_DIR))
-
+# Load configuration from config.json
 try:
-    from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN
-except ImportError:
-    print("Error: Could not import run_vllm_bench.py config.")
+    with open(CONFIG_PATH, "r") as f:
+        config_data = json.load(f)
+    MODEL_TABLE = config_data["models"]
+    DEFAULT_MODEL = config_data["default_model"]
+    MODELS_TO_RUN = list(MODEL_TABLE.keys())
+except Exception as e:
+    print(f"Error: Could not load config.json: {e}")
     sys.exit(1)
 
 if (OPT_DIR / "max_context_results.json").exists():
@@ -331,28 +330,41 @@ def main():
     check_dependencies()
     gpu_count = detect_gpus()
     
-    while True:
-        # Build Model Menu
-        menu_items = []
-        for i, m_id in enumerate(MODELS_TO_RUN):
-            name = m_id.split("/")[-1]
-            # Pre-calc verified ctx for 'default' TP to show in menu? 
-            # Or just show names. Just names is cleaner.
-            config = MODEL_TABLE[m_id]
-            menu_items.extend([str(i), name])
+    # Check if we should use default model (for docker startup)
+    use_default = os.getenv("USE_DEFAULT_MODEL", "false").lower() == "true"
+    
+    if use_default:
+        # Find the index of default model
+        try:
+            default_idx = MODELS_TO_RUN.index(DEFAULT_MODEL)
+            print(f"Using default model: {DEFAULT_MODEL}")
+            configure_and_launch(default_idx, gpu_count)
+        except ValueError:
+            print(f"Error: Default model {DEFAULT_MODEL} not found in configuration")
+            sys.exit(1)
+    else:
+        while True:
+            # Build Model Menu
+            menu_items = []
+            for i, m_id in enumerate(MODELS_TO_RUN):
+                name = m_id.split("/")[-1]
+                # Mark default model
+                if m_id == DEFAULT_MODEL:
+                    name += " (Default)"
+                menu_items.extend([str(i), name])
+                
+            choice = run_dialog([
+                "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
+                "--title", "Select Model",
+                "--menu", "Choose a model to serve:", "20", "60", "10"
+            ] + menu_items)
             
-        choice = run_dialog([
-            "--clear", "--backtitle", f"AMD R9700 vLLM Launcher (GPUs: {gpu_count})",
-            "--title", "Select Model",
-            "--menu", "Choose a model to serve:", "20", "60", "10"
-        ] + menu_items)
-        
-        if not choice:
-            subprocess.run(["clear"])
-            print("Selection cancelled.")
-            sys.exit(0)
-            
-        configure_and_launch(int(choice), gpu_count)
+            if not choice:
+                subprocess.run(["clear"])
+                print("Selection cancelled.")
+                sys.exit(0)
+                
+            configure_and_launch(int(choice), gpu_count)
 
 if __name__ == "__main__":
     main()