diff --git a/README.md b/README.md index 9f3a1fd..31fc9b8 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Once the server is up, hit the OpenAI‑compatible endpoint: ```bash curl -X POST http://localhost:8000/v1/chat/completions \ -H "Content-Type: application/json" \ - -d '{"model":"Qwen/Qwen2.5-7B-Instruct","messages":[{"role":"user","content":"Hello! Test the performance."}]}' + -d '{"model":"Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit","messages":[{"role":"user","content":"Hello! Test the performance."}]}' ``` You should receive a JSON response with a `choices[0].message.content` reply. @@ -206,4 +206,3 @@ docker run -p 3000:3000 \ -v chat-ui-data:/data \ ghcr.io/huggingface/chat-ui-db ``` - diff --git a/config.json b/config.json index 76c5b40..bd65c0b 100644 --- a/config.json +++ b/config.json @@ -35,6 +35,9 @@ "max_num_seqs": "32", "max_tokens": "65536", "gpu_util": "0.95", + "tool_call_parser": "qwen3_xml", + "enable_auto_tool_choice": false, + "served_model_name": "Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", "hf_model_id": "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit" }, "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit": { @@ -47,6 +50,8 @@ "enforce_eager": false, "env": {"VLLM_USE_TRITON_AWQ": "1"}, "tool_call_parser": "qwen3_xml", + "enable_auto_tool_choice": false, + "served_model_name": "Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", "hf_model_id": "cpatonn/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit" }, "gemma-3-27b-it-FP8-dynamic": { diff --git a/scripts/start_vllm.py b/scripts/start_vllm.py index 342ad0a..8827cc5 100644 --- a/scripts/start_vllm.py +++ b/scripts/start_vllm.py @@ -128,12 +128,14 @@ def launch_model(model_id, config, model_path, gpu_count): ctx = int(config.get("ctx", 8192)) max_seqs = int(config.get("max_num_seqs", 64)) gpu_util = float(config.get("gpu_util", 0.98)) + served_model_name = config.get("served_model_name", model_id) log(f"Config: TP={tp_size}, Ctx={ctx}, Seqs={max_seqs}, Util={gpu_util}") # Build command cmd = [ "vllm", "serve", model_path, + "--served-model-name", served_model_name, "--host", HOST, "--port", PORT, "--tensor-parallel-size", str(tp_size), @@ -159,9 +161,16 @@ def launch_model(model_id, config, model_path, gpu_count): tool_call_parser = config.get("tool_call_parser") if tool_call_parser: cmd.extend(["--tool-call-parser", tool_call_parser]) - cmd.extend(["--enable-auto-tool-choice"]) + openclaw_compat = os.getenv("OPENCLAW_COMPAT", "false").lower() == "true" + enable_auto_tool_choice = config.get("enable_auto_tool_choice") + if enable_auto_tool_choice is None: + enable_auto_tool_choice = not openclaw_compat + if enable_auto_tool_choice: + cmd.extend(["--enable-auto-tool-choice"]) + log("Added auto tool choice enabled") + else: + log("Auto tool choice disabled") log(f"Added tool call parser: {tool_call_parser}") - log("Added auto tool choice enabled") log(f"Command: {' '.join(cmd)}") diff --git a/test.py b/test.py index a96e11e..895f725 100644 --- a/test.py +++ b/test.py @@ -7,7 +7,7 @@ client = OpenAI( # 聊天完成 response = client.chat.completions.create( - model="/opt/model/Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", + model="Qwen3-Next-80B-A3B-Instruct-AWQ-4bit", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "你好,请介绍一下你自己"} @@ -16,4 +16,4 @@ response = client.chat.completions.create( temperature=0.7 ) -print(response.choices[0].message.content) \ No newline at end of file +print(response.choices[0].message.content)