from functools import lru_cache import os from typing import Optional from pydantic import BaseModel from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings class Settings(BaseModel): config_file: str = "config.json" model_key: Optional[str] = None selected_model: Optional[str] = None model_name: str = "" served_model_name: Optional[str] = None host: str = "0.0.0.0" port: int = 8000 openai_host: str = "0.0.0.0" openai_port: int = 8001 public_model_name: str = "Qwen_local_model" model_root: str = "/opt/model" offline_mode: bool = True max_model_len: int = 8192 gpu_memory_utilization: float = 0.92 tensor_parallel_size: int = 2 max_num_seqs: int = 64 max_tokens: int = 4096 dtype: str = "bfloat16" enforce_eager: bool = False trust_remote_code: bool = False tool_call_parser: Optional[str] = None enable_auto_tool_choice: bool = False revision: Optional[str] = None api_key: Optional[str] = None @lru_cache(maxsize=1) def get_settings() -> Settings: catalog = load_catalog("config.json") runtime = resolve_runtime_settings(catalog) settings = Settings( config_file="config.json", model_key=runtime["model_key"], host=runtime["host"], port=runtime["port"], openai_host=runtime["openai_host"], openai_port=runtime["openai_port"], public_model_name=runtime["public_model_name"], model_root=runtime["model_root"], offline_mode=runtime["offline_mode"], api_key=runtime["api_key"], tensor_parallel_size=runtime["tensor_parallel_size"], dtype=runtime["dtype"], revision=runtime["revision"], ) _, updates, env_vars = resolve_model_profile( content=catalog, requested_model=settings.model_key, requested_tp=settings.tensor_parallel_size, ) for key, value in env_vars.items(): os.environ[key] = value return settings.model_copy(update=updates | runtime)