189 lines
8.1 KiB
Python
189 lines
8.1 KiB
Python
import json
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def _to_bool(value: Any, default: bool) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
normalized = value.strip().lower()
|
|
if normalized in {"true", "1", "yes", "y"}:
|
|
return True
|
|
if normalized in {"false", "0", "no", "n"}:
|
|
return False
|
|
return default
|
|
|
|
|
|
def _to_int(value: Any, default: int) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _to_float(value: Any, default: float) -> float:
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _to_str(value: Any, default: str = "") -> str:
|
|
return str(value).strip() if value is not None else default
|
|
|
|
|
|
def _join_posix(base_path: str, suffix_path: str) -> str:
|
|
return f"{base_path.rstrip('/')}/{suffix_path.lstrip('/')}"
|
|
|
|
|
|
def _resolve_profile_model_path(profile: dict[str, Any], model_root: str, model_key: str) -> str:
|
|
local_path = _to_str(profile.get("local_path"))
|
|
if not local_path:
|
|
raise ValueError(f"model profile '{model_key}' must provide local_path")
|
|
if "://" in local_path:
|
|
raise ValueError(f"model profile '{model_key}' local_path must be local filesystem path")
|
|
resolved = local_path
|
|
if not local_path.startswith("/"):
|
|
if not model_root:
|
|
raise ValueError("config.json model_root cannot be empty when local_path is relative")
|
|
resolved = _join_posix(model_root, local_path)
|
|
return resolved
|
|
|
|
|
|
def load_catalog(catalog_path: str = "config.json") -> dict[str, Any]:
|
|
content = json.loads(Path(catalog_path).read_text(encoding="utf-8"))
|
|
if not isinstance(content, dict):
|
|
raise ValueError("config.json must be a JSON object")
|
|
return content
|
|
|
|
|
|
def resolve_runtime_settings(content: dict[str, Any]) -> dict[str, Any]:
|
|
services = content.get("services", {})
|
|
api_service = dict(services.get("api", {}))
|
|
openai_service = dict(services.get("openai", {}))
|
|
models = dict(content.get("models", {}))
|
|
openai_port = _to_int(openai_service.get("port"), 8001)
|
|
internal_url = _to_str(content.get("vllm_openai_internal_url"))
|
|
if not internal_url:
|
|
internal_url = f"http://127.0.0.1:{openai_port}/v1"
|
|
enable_thinking_env = os.getenv("VLLM_ENABLE_THINKING")
|
|
if enable_thinking_env is not None:
|
|
default_enable_thinking = _to_bool(enable_thinking_env, False)
|
|
else:
|
|
default_enable_thinking = _to_bool(content.get("default_enable_thinking"), False)
|
|
|
|
reasoning_enabled_env = os.getenv("VLLM_REASONING_ENABLED")
|
|
if reasoning_enabled_env is not None:
|
|
reasoning_enabled = _to_bool(reasoning_enabled_env, False)
|
|
else:
|
|
reasoning_enabled = _to_bool(content.get("reasoning_enabled"), False)
|
|
|
|
return {
|
|
"host": str(api_service.get("host", "0.0.0.0")),
|
|
"port": _to_int(api_service.get("port"), 8000),
|
|
"openai_host": str(openai_service.get("host", "0.0.0.0")),
|
|
"openai_port": openai_port,
|
|
"vllm_openai_internal_url": internal_url.rstrip("/"),
|
|
"public_model_name": _to_str(content.get("public_model_name"), "Qwen_local_model"),
|
|
"default_enable_thinking": default_enable_thinking,
|
|
"api_key": str(content.get("api_key", "")).strip() or None,
|
|
"reasoning_enabled": reasoning_enabled,
|
|
"tensor_parallel_size": _to_int(content.get("tensor_parallel_size"), 2),
|
|
"dtype": str(content.get("dtype", "bfloat16")),
|
|
"revision": str(content.get("revision", "")).strip() or None,
|
|
"model_root": _to_str(content.get("model_root"), "/opt/model"),
|
|
"offline_mode": True,
|
|
"model_key": str(models.get("selected", "")).strip() or None,
|
|
}
|
|
|
|
|
|
def resolve_model_profile(
|
|
content: dict[str, Any], requested_model: str | None, requested_tp: int
|
|
) -> tuple[str, dict[str, Any], dict[str, str]]:
|
|
models = dict(content.get("models", {}))
|
|
profiles = dict(models.get("profiles", {}))
|
|
default_model = models.get("default")
|
|
model_key = requested_model or default_model
|
|
if not model_key or model_key not in profiles:
|
|
raise ValueError(f"model profile '{model_key}' not found in config.json")
|
|
profile = profiles[model_key]
|
|
if not isinstance(profile, dict):
|
|
raise ValueError(f"model profile '{model_key}' must be a JSON object")
|
|
model_root = _to_str(content.get("model_root"), "/opt/model")
|
|
valid_tp_raw = profile.get("valid_tp", [])
|
|
valid_tp = [_to_int(item, 0) for item in valid_tp_raw if _to_int(item, 0) > 0]
|
|
resolved_tp = requested_tp
|
|
if valid_tp and resolved_tp not in valid_tp:
|
|
resolved_tp = valid_tp[0]
|
|
speculative = dict(profile.get("speculative", {}))
|
|
speculative_method = _to_str(speculative.get("method"))
|
|
speculative_model_path = _to_str(speculative.get("model"))
|
|
num_speculative_tokens = _to_int(speculative.get("num_speculative_tokens"), 0)
|
|
speculative_draft_tp = _to_int(speculative.get("draft_tensor_parallel_size"), 0)
|
|
if speculative_method and speculative_model_path:
|
|
resolved_speculative_model = speculative_model_path
|
|
if not speculative_model_path.startswith("/"):
|
|
if not model_root:
|
|
raise ValueError("config.json model_root cannot be empty when speculative model path is relative")
|
|
resolved_speculative_model = _join_posix(model_root, speculative_model_path)
|
|
else:
|
|
resolved_speculative_model = ""
|
|
updates = {
|
|
"selected_model": model_key,
|
|
"model_name": _resolve_profile_model_path(profile, model_root, model_key),
|
|
"served_model_name": profile.get("served_model_name", model_key),
|
|
"dtype": _to_str(profile.get("dtype")),
|
|
"quantization": _to_str(profile.get("quantization")),
|
|
"model_impl": _to_str(profile.get("model_impl")),
|
|
"reasoning_parser": _to_str(os.getenv("VLLM_REASONING_PARSER") or profile.get("reasoning_parser")),
|
|
"max_model_len": _to_int(profile.get("ctx"), 8192),
|
|
"max_num_seqs": _to_int(profile.get("max_num_seqs"), 64),
|
|
"max_tokens": _to_int(profile.get("max_tokens"), 4096),
|
|
"gpu_memory_utilization": _to_float(profile.get("gpu_util"), 0.92),
|
|
"trust_remote_code": _to_bool(profile.get("trust_remote"), False),
|
|
"enforce_eager": _to_bool(profile.get("enforce_eager"), False),
|
|
"tensor_parallel_size": resolved_tp,
|
|
"tool_call_parser": profile.get("tool_call_parser"),
|
|
"enable_auto_tool_choice": _to_bool(profile.get("enable_auto_tool_choice"), False),
|
|
"kv_cache_dtype": _to_str(profile.get("kv_cache_dtype")),
|
|
"enable_prefix_caching": _to_bool(profile.get("enable_prefix_caching"), False),
|
|
"max_num_batched_tokens": _to_int(profile.get("max_num_batched_tokens"), 0),
|
|
"language_model_only": _to_bool(profile.get("language_model_only"), False),
|
|
"speculative_method": speculative_method,
|
|
"speculative_model": resolved_speculative_model,
|
|
"num_speculative_tokens": num_speculative_tokens,
|
|
"speculative_draft_tp": speculative_draft_tp,
|
|
}
|
|
env_vars = {str(k): str(v) for k, v in dict(profile.get("env", {})).items()}
|
|
env_vars["HF_HUB_OFFLINE"] = "1"
|
|
env_vars["TRANSFORMERS_OFFLINE"] = "1"
|
|
if "VLLM_RPC_TIMEOUT" not in env_vars:
|
|
env_vars["VLLM_RPC_TIMEOUT"] = "300"
|
|
if "VLLM_WORKER_MULTIPROC_METHOD" not in env_vars:
|
|
env_vars["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
return model_key, updates, env_vars
|
|
|
|
|
|
def load_app_config(config_file: str = "config.json") -> dict[str, Any]:
|
|
"""
|
|
统一的应用配置加载函数，封装完整的配置加载流程。
|
|
|
|
Args:
|
|
config_file: 配置文件路径，默认为 "config.json"
|
|
|
|
Returns:
|
|
包含合并后配置的字典，包括 runtime settings 和 model profile updates
|
|
"""
|
|
catalog = load_catalog(config_file)
|
|
runtime = resolve_runtime_settings(catalog)
|
|
_, updates, env_vars = resolve_model_profile(
|
|
content=catalog,
|
|
requested_model=runtime["model_key"],
|
|
requested_tp=runtime["tensor_parallel_size"],
|
|
)
|
|
for key, value in env_vars.items():
|
|
os.environ[key] = value
|
|
return {**runtime, **updates}
|