x

app/model_catalog.py

@@ -121,14 +121,15 @@ def resolve_model_profile(
         "tensor_parallel_size": resolved_tp,
         "tool_call_parser": profile.get("tool_call_parser"),
         "enable_auto_tool_choice": _to_bool(profile.get("enable_auto_tool_choice"), False),
+        "kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")),
+        "enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
+        "max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
     }
     env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
     env_vars["HF_HUB_OFFLINE"] = "1"
     env_vars["TRANSFORMERS_OFFLINE"] = "1"
-    # 添加 vLLM RPC 超时环境变量，防止共享内存通信超时
     if "VLLM_RPC_TIMEOUT" not in env_vars:
         env_vars["VLLM_RPC_TIMEOUT"] = "300"
-    # 添加 vLLM 工作进程超时环境变量
     if "VLLM_WORKER_MULTIPROC_METHOD" not in env_vars:
         env_vars["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
     return model_key, updates, env_vars

app/start_openai.py

@@ -75,6 +75,12 @@ def build_command() -> list[str]:
         cmd.extend(["--revision", revision])
     if api_key:
         cmd.extend(["--api-key", api_key])
+    if updates.get("kv_cache_dtype"):
+        cmd.extend(["--kv-cache-dtype", str(updates["kv_cache_dtype"])])
+    if updates.get("enable_prefix_caching"):
+        cmd.append("--enable-prefix-caching")
+    if updates.get("max_num_batched_tokens", 0) > 0:
+        cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])])
     return cmd
 
 

config.json

@@ -20,7 +20,7 @@
   "revision": "",
   "models": {
     "default": "Qwen3.5-35B-A3B-GPTQ-Int4",
-    "selected": "Qwen3.5-27B-FP8",
+    "selected": "Gemma-4-26B-A4B",
     "profiles": {
       "GLM-4.7-Flash-AWQ": {
         "local_path": "GLM-4.7-Flash-AWQ",
@@ -95,7 +95,7 @@
         "ctx": "131072",
         "trust_remote": true,
         "valid_tp": [2],
-        "max_num_seqs": "6",
+        "max_num_seqs": "12",
         "max_tokens": "8096",
         "gpu_util": "0.90",
         "enforce_eager": false,
@@ -104,6 +104,22 @@
         "enable_auto_tool_choice": true,
         "served_model_name": "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
         "hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit"
+      },
+      "Gemma-4-26B-A4B": {
+        "local_path": "gemma-4-26B-A4B",
+        "dtype": "auto",
+        "ctx": "65536",
+        "trust_remote": true,
+        "valid_tp": [1, 2],
+        "max_num_seqs": 16,
+        "max_tokens": "8192",
+        "enforce_eager": false,
+        "gpu_util": "0.85",
+        "kv_cache_dtype": "fp8",
+        "enable_prefix_caching": true,
+        "max_num_batched_tokens": 16384,
+        "served_model_name": "Gemma-4-26B-A4B",
+        "hf_model_id": "google/gemma-4-26B-A4B"
       }
     }
   }

docker-compose.yml

@@ -14,6 +14,11 @@ services:
       - ./config.json:/workspace/config.json:ro
     environment:
       HF_HOME: /opt/model
+      # 严格离线模式，禁止任何网络下载
+      HF_HUB_OFFLINE: "1"
+      TRANSFORMERS_OFFLINE: "1"
+      HF_DATASETS_OFFLINE: "1"
+      # vLLM 相关配置
       VLLM_RPC_TIMEOUT: "300"
       VLLM_WORKER_MULTIPROC_METHOD: "spawn"
       PYTHONUNBUFFERED: "1"