99 lines
3.5 KiB
Python
99 lines
3.5 KiB
Python
import json
|
|
import subprocess
|
|
import sys
|
|
|
|
from app.model_catalog import load_app_config
|
|
|
|
|
|
def build_command() -> list[str]:
|
|
config = load_app_config("config.json")
|
|
host = str(config["openai_host"])
|
|
port = str(config["openai_port"])
|
|
public_model_name = str(config["public_model_name"]).strip()
|
|
default_enable_thinking = bool(config["default_enable_thinking"])
|
|
reasoning_enabled = bool(config["reasoning_enabled"])
|
|
api_key = config.get("api_key") or ""
|
|
dtype = str(config.get("dtype", "bfloat16"))
|
|
quantization = str(config.get("quantization", "")).strip()
|
|
model_impl = str(config.get("model_impl", "")).strip()
|
|
reasoning_parser = str(config.get("reasoning_parser", "")).strip()
|
|
revision = config.get("revision", "") or ""
|
|
cmd = [
|
|
sys.executable,
|
|
"-m",
|
|
"vllm.entrypoints.openai.api_server",
|
|
"--host",
|
|
host,
|
|
"--port",
|
|
port,
|
|
"--model",
|
|
str(config["model_name"]),
|
|
"--served-model-name",
|
|
public_model_name or str(config["served_model_name"]),
|
|
"--tensor-parallel-size",
|
|
str(config["tensor_parallel_size"]),
|
|
"--max-model-len",
|
|
str(config["max_model_len"]),
|
|
"--gpu-memory-utilization",
|
|
str(config["gpu_memory_utilization"]),
|
|
"--max-num-seqs",
|
|
str(config["max_num_seqs"]),
|
|
"--dtype",
|
|
dtype,
|
|
]
|
|
if config["trust_remote_code"]:
|
|
cmd.append("--trust-remote-code")
|
|
if config["enforce_eager"]:
|
|
cmd.append("--enforce-eager")
|
|
if config["enable_auto_tool_choice"]:
|
|
cmd.append("--enable-auto-tool-choice")
|
|
if config["tool_call_parser"]:
|
|
cmd.extend(["--tool-call-parser", str(config["tool_call_parser"])])
|
|
cmd.extend(
|
|
[
|
|
"--default-chat-template-kwargs",
|
|
json.dumps({"enable_thinking": default_enable_thinking}),
|
|
]
|
|
)
|
|
if reasoning_enabled and reasoning_parser:
|
|
cmd.extend(["--reasoning-parser", reasoning_parser])
|
|
if quantization:
|
|
cmd.extend(["--quantization", quantization])
|
|
if model_impl:
|
|
cmd.extend(["--model-impl", model_impl])
|
|
if revision:
|
|
cmd.extend(["--revision", revision])
|
|
if api_key:
|
|
cmd.extend(["--api-key", api_key])
|
|
if config.get("kv_cache_dtype"):
|
|
cmd.extend(["--kv-cache-dtype", str(config["kv_cache_dtype"])])
|
|
if config.get("enable_prefix_caching"):
|
|
cmd.append("--enable-prefix-caching")
|
|
if config.get("max_num_batched_tokens", 0) > 0:
|
|
cmd.extend(["--max-num-batched-tokens", str(config["max_num_batched_tokens"])])
|
|
if updates.get("language_model_only"):
|
|
cmd.append("--language-model-only")
|
|
speculative_method = str(updates.get("speculative_method") or "").strip()
|
|
speculative_model = str(updates.get("speculative_model") or "").strip()
|
|
num_speculative_tokens = int(updates.get("num_speculative_tokens") or 0)
|
|
speculative_draft_tp = int(updates.get("speculative_draft_tp") or 0)
|
|
if speculative_method and speculative_model and num_speculative_tokens > 0:
|
|
spec_config: dict = {
|
|
"method": speculative_method,
|
|
"model": speculative_model,
|
|
"num_speculative_tokens": num_speculative_tokens,
|
|
}
|
|
if speculative_draft_tp > 0:
|
|
spec_config["draft_tensor_parallel_size"] = speculative_draft_tp
|
|
cmd.extend(["--speculative-config", json.dumps(spec_config)])
|
|
return cmd
|
|
|
|
|
|
def main() -> None:
|
|
command = build_command()
|
|
raise SystemExit(subprocess.call(command))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|