diff --git a/app/model_catalog.py b/app/model_catalog.py index af91446..4166fd8 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -124,6 +124,7 @@ def resolve_model_profile( "kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")), "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False), "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0), + "language_model_only": _to_bool(profile.get("language_model_only"), False), } env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()} env_vars["HF_HUB_OFFLINE"] = "1" diff --git a/app/start_openai.py b/app/start_openai.py index 5293ace..97545a5 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -81,6 +81,8 @@ def build_command() -> list[str]: cmd.append("--enable-prefix-caching") if updates.get("max_num_batched_tokens", 0) > 0: cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])]) + if updates.get("language_model_only"): + cmd.append("--language-model-only") return cmd diff --git a/config.json b/config.json index 5be4c96..9d0d8ab 100644 --- a/config.json +++ b/config.json @@ -38,12 +38,13 @@ "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "enable_auto_tool_choice": true, + "language_model_only": true, "served_model_name": "Qwen3.6-35B-A3B-FP8", "hf_model_id": "Qwen/Qwen3.6-35B-A3B-FP8" }, "Qwen3.6-27B": { "local_path": "Qwen3.6-27B", - "ctx": "262144", + "ctx": "131072", "max_tokens": "32768", "max_num_batched_tokens": 16384, "trust_remote": true, @@ -53,12 +54,13 @@ 2 ], "max_num_seqs": 32, - "gpu_util": "0.90", + "gpu_util": "0.92", "kv_cache_dtype": "fp8", "enable_prefix_caching": true, "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "enable_auto_tool_choice": true, + "language_model_only": true, "served_model_name": "Qwen3.6-27B", "hf_model_id": "Qwen/Qwen3.6-27B" },