83 lines
3.3 KiB
Python
83 lines
3.3 KiB
Python
from functools import lru_cache
|
|
from typing import Optional
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from app.model_catalog import load_app_config
|
|
|
|
|
|
class Settings(BaseModel):
|
|
config_file: str = "config.json"
|
|
model_key: Optional[str] = None
|
|
selected_model: Optional[str] = None
|
|
model_name: str = ""
|
|
served_model_name: Optional[str] = None
|
|
host: str = "0.0.0.0"
|
|
port: int = 8000
|
|
openai_host: str = "0.0.0.0"
|
|
openai_port: int = 8001
|
|
vllm_openai_internal_url: str = "http://127.0.0.1:8001/v1"
|
|
public_model_name: str = "Qwen_local_model"
|
|
default_enable_thinking: bool = False
|
|
reasoning_enabled: bool = False
|
|
model_root: str = "/opt/model"
|
|
offline_mode: bool = True
|
|
max_model_len: int = 8192
|
|
gpu_memory_utilization: float = 0.92
|
|
tensor_parallel_size: int = 2
|
|
max_num_seqs: int = 64
|
|
max_tokens: int = 4096
|
|
dtype: str = "bfloat16"
|
|
enforce_eager: bool = False
|
|
trust_remote_code: bool = False
|
|
tool_call_parser: Optional[str] = None
|
|
enable_auto_tool_choice: bool = False
|
|
revision: Optional[str] = None
|
|
api_key: Optional[str] = None
|
|
quantization: Optional[str] = None
|
|
model_impl: Optional[str] = None
|
|
reasoning_parser: Optional[str] = None
|
|
kv_cache_dtype: Optional[str] = None
|
|
enable_prefix_caching: bool = False
|
|
max_num_batched_tokens: int = 0
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_settings() -> Settings:
|
|
config = load_app_config("config.json")
|
|
return Settings(
|
|
config_file="config.json",
|
|
model_key=config.get("model_key"),
|
|
selected_model=config.get("selected_model"),
|
|
model_name=config.get("model_name", ""),
|
|
served_model_name=config.get("served_model_name"),
|
|
host=config.get("host", "0.0.0.0"),
|
|
port=config.get("port", 8000),
|
|
openai_host=config.get("openai_host", "0.0.0.0"),
|
|
openai_port=config.get("openai_port", 8001),
|
|
vllm_openai_internal_url=config.get("vllm_openai_internal_url", "http://127.0.0.1:8001/v1"),
|
|
public_model_name=config.get("public_model_name", "Qwen_local_model"),
|
|
default_enable_thinking=config.get("default_enable_thinking", False),
|
|
reasoning_enabled=config.get("reasoning_enabled", False),
|
|
model_root=config.get("model_root", "/opt/model"),
|
|
offline_mode=config.get("offline_mode", True),
|
|
max_model_len=config.get("max_model_len", 8192),
|
|
gpu_memory_utilization=config.get("gpu_memory_utilization", 0.92),
|
|
tensor_parallel_size=config.get("tensor_parallel_size", 2),
|
|
max_num_seqs=config.get("max_num_seqs", 64),
|
|
max_tokens=config.get("max_tokens", 4096),
|
|
dtype=config.get("dtype", "bfloat16"),
|
|
enforce_eager=config.get("enforce_eager", False),
|
|
trust_remote_code=config.get("trust_remote_code", False),
|
|
tool_call_parser=config.get("tool_call_parser"),
|
|
enable_auto_tool_choice=config.get("enable_auto_tool_choice", False),
|
|
revision=config.get("revision"),
|
|
api_key=config.get("api_key"),
|
|
quantization=config.get("quantization"),
|
|
model_impl=config.get("model_impl"),
|
|
reasoning_parser=config.get("reasoning_parser"),
|
|
kv_cache_dtype=config.get("kv_cache_dtype"),
|
|
enable_prefix_caching=config.get("enable_prefix_caching", False),
|
|
max_num_batched_tokens=config.get("max_num_batched_tokens", 0),
|
|
)
|