x

2026-03-29 05:47:30 +08:00
parent dd5afbe6d0
commit 23909c205e
5 changed files with 46 additions and 7 deletions
@@ -17,6 +17,8 @@ class Settings(BaseModel):
    port: int = 8000
    openai_host: str = "0.0.0.0"
    openai_port: int = 8001
+    model_root: str = "/opt/model"
+    offline_mode: bool = True
    max_model_len: int = 8192
    gpu_memory_utilization: float = 0.92
    tensor_parallel_size: int = 2
@@ -42,6 +44,8 @@ def get_settings() -> Settings:
        port=runtime["port"],
        openai_host=runtime["openai_host"],
        openai_port=runtime["openai_port"],
+        model_root=runtime["model_root"],
+        offline_mode=runtime["offline_mode"],
        api_key=runtime["api_key"],
        tensor_parallel_size=runtime["tensor_parallel_size"],
        dtype=runtime["dtype"],