69 lines
2.0 KiB
Python
69 lines
2.0 KiB
Python
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from app.model_catalog import resolve_model_profile
|
|
|
|
|
|
def build_command() -> list[str]:
|
|
config_file = os.getenv("MODEL_CONFIG_FILE", "config.json")
|
|
model_key = os.getenv("MODEL_KEY")
|
|
requested_tp = int(os.getenv("TENSOR_PARALLEL_SIZE", "2"))
|
|
_, updates, env_vars = resolve_model_profile(
|
|
catalog_path=config_file,
|
|
requested_model=model_key,
|
|
requested_tp=requested_tp,
|
|
)
|
|
for key, value in env_vars.items():
|
|
os.environ[key] = value
|
|
host = os.getenv("OPENAI_HOST", "0.0.0.0")
|
|
port = os.getenv("OPENAI_PORT", "8001")
|
|
dtype = os.getenv("DTYPE", "bfloat16")
|
|
revision = os.getenv("REVISION", "").strip()
|
|
api_key = os.getenv("API_KEY", "").strip()
|
|
cmd = [
|
|
sys.executable,
|
|
"-m",
|
|
"vllm.entrypoints.openai.api_server",
|
|
"--host",
|
|
host,
|
|
"--port",
|
|
port,
|
|
"--model",
|
|
str(updates["model_name"]),
|
|
"--served-model-name",
|
|
str(updates["served_model_name"]),
|
|
"--tensor-parallel-size",
|
|
str(updates["tensor_parallel_size"]),
|
|
"--max-model-len",
|
|
str(updates["max_model_len"]),
|
|
"--gpu-memory-utilization",
|
|
str(updates["gpu_memory_utilization"]),
|
|
"--max-num-seqs",
|
|
str(updates["max_num_seqs"]),
|
|
"--dtype",
|
|
dtype,
|
|
]
|
|
if updates["trust_remote_code"]:
|
|
cmd.append("--trust-remote-code")
|
|
if updates["enforce_eager"]:
|
|
cmd.append("--enforce-eager")
|
|
if updates["enable_auto_tool_choice"]:
|
|
cmd.append("--enable-auto-tool-choice")
|
|
if updates["tool_call_parser"]:
|
|
cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
|
|
if revision:
|
|
cmd.extend(["--revision", revision])
|
|
if api_key:
|
|
cmd.extend(["--api-key", api_key])
|
|
return cmd
|
|
|
|
|
|
def main() -> None:
|
|
command = build_command()
|
|
raise SystemExit(subprocess.call(command))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|