x

app/model_catalog.py

@@ -125,4 +125,10 @@ def resolve_model_profile(
     env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
     env_vars["HF_HUB_OFFLINE"] = "1"
     env_vars["TRANSFORMERS_OFFLINE"] = "1"
+    # 添加 vLLM RPC 超时环境变量，防止共享内存通信超时
+    if "VLLM_RPC_TIMEOUT" not in env_vars:
+        env_vars["VLLM_RPC_TIMEOUT"] = "300"
+    # 添加 vLLM 工作进程超时环境变量
+    if "VLLM_WORKER_MULTIPROC_METHOD" not in env_vars:
+        env_vars["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
     return model_key, updates, env_vars

config.json

@@ -73,18 +73,21 @@
         "local_path": "Qwen3.5-35B-A3B-GPTQ-Int4",
         "dtype": "float16",
         "quantization": "gptq",
-        "ctx": "131072",
+        "ctx": "65536",
         "trust_remote": true,
         "valid_tp": [1, 2],
-        "max_num_seqs": "8",
-        "max_tokens": "8192",
-        "gpu_util": "0.90",
+        "max_num_seqs": "4",
+        "max_tokens": "4096",
+        "gpu_util": "0.85",
         "enforce_eager": false,
         "tool_call_parser": "qwen3_coder",
         "reasoning_parser": "qwen3",
         "enable_auto_tool_choice": true,
         "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",
-        "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4"
+        "hf_model_id": "Qwen/Qwen3.5-35B-A3B-GPTQ-Int4",
+        "env": {
+          "VLLM_RPC_TIMEOUT": "300"
+        }
       },
       "Qwen3.5-35B-A3B-FP8": {
         "local_path": "Qwen3.5-35B-A3B-FP8",

docker-compose.yml

@@ -14,6 +14,9 @@ services:
       - ./config.json:/workspace/config.json:ro
     environment:
       HF_HOME: /opt/model
+      VLLM_RPC_TIMEOUT: "300"
+      VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      PYTHONUNBUFFERED: "1"
     devices:
       - /dev/kfd
       - /dev/dri