x

app/model_catalog.py

@@ -124,6 +124,7 @@ def resolve_model_profile(
         "kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")),
         "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
         "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
+        "language_model_only": _to_bool(profile.get("language_model_only"), False),
     }
     env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
     env_vars["HF_HUB_OFFLINE"] = "1"

app/start_openai.py

@@ -81,6 +81,8 @@ def build_command() -> list[str]:
         cmd.append("--enable-prefix-caching")
     if updates.get("max_num_batched_tokens", 0) > 0:
         cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])])
+    if updates.get("language_model_only"):
+        cmd.append("--language-model-only")
     return cmd