x

config.json

@@ -46,6 +46,7 @@
       "gpu_util": "0.98",
       "enforce_eager": false,
       "env": {"VLLM_USE_TRITON_AWQ": "1"},
+      "tool_call_parser": "qwen",
       "hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit"
     },
     "gemma-3-27b-it-FP8-dynamic": {

scripts/start_vllm.py

@@ -155,6 +155,12 @@ def launch_model(model_id, config, model_path, gpu_count):
         cmd.extend(["--reasoning-parser", "qwen3"])
         log("Added Qwen3.5 specific parameters: --quantization moe_wna16 --reasoning-parser qwen3")
     
+    # Add tool call parser if specified
+    tool_call_parser = config.get("tool_call_parser")
+    if tool_call_parser:
+        cmd.extend(["--tool-call-parser", tool_call_parser])
+        log(f"Added tool call parser: {tool_call_parser}")
+    
     log(f"Command: {' '.join(cmd)}")
     
     # Set environment

test.py

@@ -0,0 +1,19 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://192.168.0.11:8000/v1",
+    api_key="dummy"  # vLLM 不需要真实的 API key
+)
+
+# 聊天完成
+response = client.chat.completions.create(
+    model="/opt/model/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "你好，请介绍一下你自己"}
+    ],
+    max_tokens=500,
+    temperature=0.7
+)
+
+print(response.choices[0].message.content)