70 lines
2.3 KiB
Python
70 lines
2.3 KiB
Python
from functools import lru_cache
|
|
import os
|
|
from typing import Optional
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
|
|
|
|
|
|
class Settings(BaseModel):
|
|
config_file: str = "config.json"
|
|
model_key: Optional[str] = None
|
|
selected_model: Optional[str] = None
|
|
model_name: str = ""
|
|
served_model_name: Optional[str] = None
|
|
host: str = "0.0.0.0"
|
|
port: int = 8000
|
|
openai_host: str = "0.0.0.0"
|
|
openai_port: int = 8001
|
|
vllm_openai_internal_url: str = "http://127.0.0.1:8001/v1"
|
|
public_model_name: str = "Qwen_local_model"
|
|
default_enable_thinking: bool = False
|
|
reasoning_enabled: bool = False
|
|
model_root: str = "/opt/model"
|
|
offline_mode: bool = True
|
|
max_model_len: int = 8192
|
|
gpu_memory_utilization: float = 0.92
|
|
tensor_parallel_size: int = 2
|
|
max_num_seqs: int = 64
|
|
max_tokens: int = 4096
|
|
dtype: str = "bfloat16"
|
|
enforce_eager: bool = False
|
|
trust_remote_code: bool = False
|
|
tool_call_parser: Optional[str] = None
|
|
enable_auto_tool_choice: bool = False
|
|
revision: Optional[str] = None
|
|
api_key: Optional[str] = None
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_settings() -> Settings:
|
|
catalog = load_catalog("config.json")
|
|
runtime = resolve_runtime_settings(catalog)
|
|
settings = Settings(
|
|
config_file="config.json",
|
|
model_key=runtime["model_key"],
|
|
host=runtime["host"],
|
|
port=runtime["port"],
|
|
openai_host=runtime["openai_host"],
|
|
openai_port=runtime["openai_port"],
|
|
vllm_openai_internal_url=runtime["vllm_openai_internal_url"],
|
|
public_model_name=runtime["public_model_name"],
|
|
default_enable_thinking=runtime["default_enable_thinking"],
|
|
reasoning_enabled=runtime["reasoning_enabled"],
|
|
model_root=runtime["model_root"],
|
|
offline_mode=runtime["offline_mode"],
|
|
api_key=runtime["api_key"],
|
|
tensor_parallel_size=runtime["tensor_parallel_size"],
|
|
dtype=runtime["dtype"],
|
|
revision=runtime["revision"],
|
|
)
|
|
_, updates, env_vars = resolve_model_profile(
|
|
content=catalog,
|
|
requested_model=settings.model_key,
|
|
requested_tp=settings.tensor_parallel_size,
|
|
)
|
|
for key, value in env_vars.items():
|
|
os.environ[key] = value
|
|
return settings.model_copy(update=updates | runtime)
|