x

app/start_openai.py

@@ -2,25 +2,25 @@ import os
 import subprocess
 import sys
 
-from app.model_catalog import resolve_model_profile
+from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
 
 
 def build_command() -> list[str]:
-    config_file = os.getenv("MODEL_CONFIG_FILE", "config.json")
-    model_key = os.getenv("MODEL_KEY")
-    requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2"))
+    config_file = "config.json"
+    catalog = load_catalog(config_file)
+    runtime = resolve_runtime_settings(catalog)
     _, updates, env_vars = resolve_model_profile(
-        catalog_path=config_file,
-        requested_model=model_key,
-        requested_tp=requested_tp,
+        content=catalog,
+        requested_model=runtime["model_key"],
+        requested_tp=runtime["tensor_parallel_size"],
     )
     for key, value in env_vars.items():
         os.environ[key] = value
-    host = os.getenv("OPENAI_HOST", "0.0.0.0")
-    port = os.getenv("OPENAI_PORT", "8001")
-    dtype = os.getenv("DTYPE", "bfloat16")
-    revision = os.getenv("REVISION", "").strip()
-    api_key = os.getenv("API_KEY", "").strip()
+    host = str(runtime["openai_host"])
+    port = str(runtime["openai_port"])
+    dtype = str(runtime["dtype"])
+    revision = runtime["revision"] or ""
+    api_key = runtime["api_key"] or ""
     cmd = [
         sys.executable,
         "-m",