x

2026-05-18 14:50:41 +08:00
parent 2d0b2f4aae
commit 4b02041c1d
3 changed files with 38 additions and 2 deletions
@@ -83,6 +83,19 @@ def build_command() -> list[str]:
        cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])])
    if updates.get("language_model_only"):
        cmd.append("--language-model-only")
+    speculative_method = str(updates.get("speculative_method") or "").strip()
+    speculative_model = str(updates.get("speculative_model") or "").strip()
+    num_speculative_tokens = int(updates.get("num_speculative_tokens") or 0)
+    speculative_draft_tp = int(updates.get("speculative_draft_tp") or 0)
+    if speculative_method and speculative_model and num_speculative_tokens > 0:
+        spec_config: dict = {
+            "method": speculative_method,
+            "model": speculative_model,
+            "num_speculative_tokens": num_speculative_tokens,
+        }
+        if speculative_draft_tp > 0:
+            spec_config["draft_tensor_parallel_size"] = speculative_draft_tp
+        cmd.extend(["--speculative-config", json.dumps(spec_config)])
    return cmd