x
This commit is contained in:
@@ -18,6 +18,7 @@ class Settings(BaseModel):
|
||||
openai_host: str = "0.0.0.0"
|
||||
openai_port: int = 8001
|
||||
public_model_name: str = "Qwen_local_model"
|
||||
reasoning_enabled: bool = False
|
||||
model_root: str = "/opt/model"
|
||||
offline_mode: bool = True
|
||||
max_model_len: int = 8192
|
||||
@@ -46,6 +47,7 @@ def get_settings() -> Settings:
|
||||
openai_host=runtime["openai_host"],
|
||||
openai_port=runtime["openai_port"],
|
||||
public_model_name=runtime["public_model_name"],
|
||||
reasoning_enabled=runtime["reasoning_enabled"],
|
||||
model_root=runtime["model_root"],
|
||||
offline_mode=runtime["offline_mode"],
|
||||
api_key=runtime["api_key"],
|
||||
|
||||
@@ -70,6 +70,7 @@ def resolve_runtime_settings(content: dict[str, Any]) -> dict[str, Any]:
|
||||
"openai_port": _to_int(openai_service.get("port"), 8001),
|
||||
"public_model_name": _to_str(content.get("public_model_name"), "Qwen_local_model"),
|
||||
"api_key": str(content.get("api_key", "")).strip() or None,
|
||||
"reasoning_enabled": _to_bool(content.get("reasoning_enabled"), False),
|
||||
"tensor_parallel_size": _to_int(content.get("tensor_parallel_size"), 2),
|
||||
"dtype": str(content.get("dtype", "bfloat16")),
|
||||
"revision": str(content.get("revision", "")).strip() or None,
|
||||
@@ -103,6 +104,7 @@ def resolve_model_profile(
|
||||
"served_model_name": profile.get("served_model_name", model_key),
|
||||
"dtype": _to_str(profile.get("dtype")),
|
||||
"quantization": _to_str(profile.get("quantization")),
|
||||
"reasoning_parser": _to_str(profile.get("reasoning_parser")),
|
||||
"max_model_len": _to_int(profile.get("ctx"), 8192),
|
||||
"max_num_seqs": _to_int(profile.get("max_num_seqs"), 64),
|
||||
"max_tokens": _to_int(profile.get("max_tokens"), 4096),
|
||||
|
||||
@@ -19,9 +19,11 @@ def build_command() -> list[str]:
|
||||
host = str(runtime["openai_host"])
|
||||
port = str(runtime["openai_port"])
|
||||
public_model_name = str(runtime["public_model_name"]).strip()
|
||||
reasoning_enabled = bool(runtime["reasoning_enabled"])
|
||||
api_key = runtime["api_key"] or ""
|
||||
dtype = str(updates["dtype"] or runtime["dtype"])
|
||||
quantization = str(updates["quantization"] or "").strip()
|
||||
reasoning_parser = str(updates["reasoning_parser"] or "").strip()
|
||||
revision = runtime["revision"] or ""
|
||||
cmd = [
|
||||
sys.executable,
|
||||
@@ -54,6 +56,8 @@ def build_command() -> list[str]:
|
||||
cmd.append("--enable-auto-tool-choice")
|
||||
if updates["tool_call_parser"]:
|
||||
cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
|
||||
if reasoning_enabled and reasoning_parser:
|
||||
cmd.extend(["--reasoning-parser", reasoning_parser])
|
||||
if quantization:
|
||||
cmd.extend(["--quantization", quantization])
|
||||
if revision:
|
||||
|
||||
Reference in New Issue
Block a user