x

app/config.py

@@ -2,44 +2,56 @@ from functools import lru_cache
 import os
 from typing import Optional
 
-from pydantic import Field
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic import BaseModel
 
-from app.model_catalog import resolve_model_profile
+from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings
 
 
-class Settings(BaseSettings):
-    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8")
-
-    config_file: str = Field(default="config.json", alias="MODEL_CONFIG_FILE")
-    model_key: Optional[str] = Field(default=None, alias="MODEL_KEY")
+class Settings(BaseModel):
+    config_file: str = "config.json"
+    model_key: Optional[str] = None
     selected_model: Optional[str] = None
-    model_name: str = Field(default="", alias="MODEL_NAME")
+    model_name: str = ""
     served_model_name: Optional[str] = None
-    host: str = Field(default="0.0.0.0", alias="HOST")
-    port: int = Field(default=8000, alias="PORT")
-    max_model_len: int = Field(default=8192, alias="MAX_MODEL_LEN")
-    gpu_memory_utilization: float = Field(default=0.92, alias="GPU_MEMORY_UTILIZATION")
-    tensor_parallel_size: int = Field(default=2, alias="TENSOR_PARALLEL_SIZE")
-    max_num_seqs: int = Field(default=64, alias="MAX_NUM_SEQS")
-    max_tokens: int = Field(default=4096, alias="MAX_TOKENS")
-    dtype: str = Field(default="bfloat16", alias="DTYPE")
-    enforce_eager: bool = Field(default=False, alias="ENFORCE_EAGER")
-    trust_remote_code: bool = Field(default=False, alias="TRUST_REMOTE_CODE")
-    tool_call_parser: Optional[str] = Field(default=None, alias="TOOL_CALL_PARSER")
-    enable_auto_tool_choice: bool = Field(default=False, alias="ENABLE_AUTO_TOOL_CHOICE")
-    revision: Optional[str] = Field(default=None, alias="REVISION")
-    api_key: Optional[str] = Field(default=None, alias="API_KEY")
+    host: str = "0.0.0.0"
+    port: int = 8000
+    openai_host: str = "0.0.0.0"
+    openai_port: int = 8001
+    max_model_len: int = 8192
+    gpu_memory_utilization: float = 0.92
+    tensor_parallel_size: int = 2
+    max_num_seqs: int = 64
+    max_tokens: int = 4096
+    dtype: str = "bfloat16"
+    enforce_eager: bool = False
+    trust_remote_code: bool = False
+    tool_call_parser: Optional[str] = None
+    enable_auto_tool_choice: bool = False
+    revision: Optional[str] = None
+    api_key: Optional[str] = None
 
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
-    settings = Settings()
+    catalog = load_catalog("config.json")
+    runtime = resolve_runtime_settings(catalog)
+    settings = Settings(
+        config_file="config.json",
+        model_key=runtime["model_key"],
+        host=runtime["host"],
+        port=runtime["port"],
+        openai_host=runtime["openai_host"],
+        openai_port=runtime["openai_port"],
+        api_key=runtime["api_key"],
+        tensor_parallel_size=runtime["tensor_parallel_size"],
+        dtype=runtime["dtype"],
+        revision=runtime["revision"],
+    )
     _, updates, env_vars = resolve_model_profile(
-        catalog_path=settings.config_file,
+        content=catalog,
         requested_model=settings.model_key,
         requested_tp=settings.tensor_parallel_size,
     )
     for key, value in env_vars.items():
         os.environ[key] = value
-    return settings.model_copy(update=updates)
+    return settings.model_copy(update=updates | runtime)