cjw/amd-r9700-vllm-toolboxes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

x

Browse Source
This commit is contained in:
SZCJW
2026-03-29 04:40:36 +08:00
parent acd5ed518e
commit 3d88c2f416
2 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.json
+2
View File
@@ -106,6 +106,7 @@
"max_num_seqs": "64", "max_num_seqs": "64",
"max_tokens": "32768", "max_tokens": "32768",
"gpu_util": "0.98", "gpu_util": "0.98",
"model_impl": "transformers",
"tool_call_parser": "qwen3_xml", "tool_call_parser": "qwen3_xml",
"enable_auto_tool_choice": true, "enable_auto_tool_choice": true,
"served_model_name": "Qwen3.5-27B-FP8", "served_model_name": "Qwen3.5-27B-FP8",
@@ -118,6 +119,7 @@
"max_num_seqs": "64", "max_num_seqs": "64",
"max_tokens": "32768", "max_tokens": "32768",
"gpu_util": "0.98", "gpu_util": "0.98",
"model_impl": "transformers",
"tool_call_parser": "qwen3_xml", "tool_call_parser": "qwen3_xml",
"enable_auto_tool_choice": true, "enable_auto_tool_choice": true,
"served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4", "served_model_name": "Qwen3.5-35B-A3B-GPTQ-Int4",
scripts/start_vllm.py
+5
View File
@@ -145,6 +145,11 @@ def launch_model(model_id, config, model_path, gpu_count):
"--dtype", "auto" "--dtype", "auto"
] ]
model_impl = config.get("model_impl") or os.getenv("VLLM_MODEL_IMPL")
if model_impl:
cmd.extend(["--model-impl", str(model_impl)])
log(f"Added model implementation: {model_impl}")
if config.get("trust_remote"): if config.get("trust_remote"):
cmd.append("--trust-remote-code") cmd.append("--trust-remote-code")