From 18a2169a52bb06b5d017dc4c138c2d0bf523961d Mon Sep 17 00:00:00 2001 From: SZCJW <792430652@qq.com> Date: Wed, 20 May 2026 01:10:08 +0800 Subject: [PATCH] x --- app/config.py | 69 ++++++++++++++++++++++++++------------------ app/model_catalog.py | 23 +++++++++++++++ app/start_openai.py | 68 +++++++++++++++++++------------------------ config.json | 4 +-- 4 files changed, 95 insertions(+), 69 deletions(-) diff --git a/app/config.py b/app/config.py index 520c61c..bc279ce 100644 --- a/app/config.py +++ b/app/config.py @@ -1,10 +1,9 @@ from functools import lru_cache -import os from typing import Optional from pydantic import BaseModel -from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings +from app.model_catalog import load_app_config class Settings(BaseModel): @@ -35,35 +34,49 @@ class Settings(BaseModel): enable_auto_tool_choice: bool = False revision: Optional[str] = None api_key: Optional[str] = None + quantization: Optional[str] = None + model_impl: Optional[str] = None + reasoning_parser: Optional[str] = None + kv_cache_dtype: Optional[str] = None + enable_prefix_caching: bool = False + max_num_batched_tokens: int = 0 @lru_cache(maxsize=1) def get_settings() -> Settings: - catalog = load_catalog("config.json") - runtime = resolve_runtime_settings(catalog) - settings = Settings( + config = load_app_config("config.json") + return Settings( config_file="config.json", - model_key=runtime["model_key"], - host=runtime["host"], - port=runtime["port"], - openai_host=runtime["openai_host"], - openai_port=runtime["openai_port"], - vllm_openai_internal_url=runtime["vllm_openai_internal_url"], - public_model_name=runtime["public_model_name"], - default_enable_thinking=runtime["default_enable_thinking"], - reasoning_enabled=runtime["reasoning_enabled"], - model_root=runtime["model_root"], - offline_mode=runtime["offline_mode"], - api_key=runtime["api_key"], - tensor_parallel_size=runtime["tensor_parallel_size"], - dtype=runtime["dtype"], - revision=runtime["revision"], + model_key=config.get("model_key"), + selected_model=config.get("selected_model"), + model_name=config.get("model_name", ""), + served_model_name=config.get("served_model_name"), + host=config.get("host", "0.0.0.0"), + port=config.get("port", 8000), + openai_host=config.get("openai_host", "0.0.0.0"), + openai_port=config.get("openai_port", 8001), + vllm_openai_internal_url=config.get("vllm_openai_internal_url", "http://127.0.0.1:8001/v1"), + public_model_name=config.get("public_model_name", "Qwen_local_model"), + default_enable_thinking=config.get("default_enable_thinking", False), + reasoning_enabled=config.get("reasoning_enabled", False), + model_root=config.get("model_root", "/opt/model"), + offline_mode=config.get("offline_mode", True), + max_model_len=config.get("max_model_len", 8192), + gpu_memory_utilization=config.get("gpu_memory_utilization", 0.92), + tensor_parallel_size=config.get("tensor_parallel_size", 2), + max_num_seqs=config.get("max_num_seqs", 64), + max_tokens=config.get("max_tokens", 4096), + dtype=config.get("dtype", "bfloat16"), + enforce_eager=config.get("enforce_eager", False), + trust_remote_code=config.get("trust_remote_code", False), + tool_call_parser=config.get("tool_call_parser"), + enable_auto_tool_choice=config.get("enable_auto_tool_choice", False), + revision=config.get("revision"), + api_key=config.get("api_key"), + quantization=config.get("quantization"), + model_impl=config.get("model_impl"), + reasoning_parser=config.get("reasoning_parser"), + kv_cache_dtype=config.get("kv_cache_dtype"), + enable_prefix_caching=config.get("enable_prefix_caching", False), + max_num_batched_tokens=config.get("max_num_batched_tokens", 0), ) - _, updates, env_vars = resolve_model_profile( - content=catalog, - requested_model=settings.model_key, - requested_tp=settings.tensor_parallel_size, - ) - for key, value in env_vars.items(): - os.environ[key] = value - return settings.model_copy(update=updates | runtime) diff --git a/app/model_catalog.py b/app/model_catalog.py index 600c31f..35d6d88 100644 --- a/app/model_catalog.py +++ b/app/model_catalog.py @@ -1,4 +1,5 @@ import json +import os from pathlib import Path from typing import Any @@ -151,3 +152,25 @@ def resolve_model_profile( if "VLLM_WORKER_MULTIPROC_METHOD" not in env_vars: env_vars["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" return model_key, updates, env_vars + + +def load_app_config(config_file: str = "config.json") -> dict[str, Any]: + """ + 统一的应用配置加载函数，封装完整的配置加载流程。 + + Args: + config_file: 配置文件路径，默认为 "config.json" + + Returns: + 包含合并后配置的字典，包括 runtime settings 和 model profile updates + """ + catalog = load_catalog(config_file) + runtime = resolve_runtime_settings(catalog) + _, updates, env_vars = resolve_model_profile( + content=catalog, + requested_model=runtime["model_key"], + requested_tp=runtime["tensor_parallel_size"], + ) + for key, value in env_vars.items(): + os.environ[key] = value + return {**runtime, **updates} diff --git a/app/start_openai.py b/app/start_openai.py index ebabdf7..ba42e73 100644 --- a/app/start_openai.py +++ b/app/start_openai.py @@ -1,33 +1,23 @@ import json -import os import subprocess import sys -from app.model_catalog import load_catalog, resolve_model_profile, resolve_runtime_settings +from app.model_catalog import load_app_config def build_command() -> list[str]: - config_file = "config.json" - catalog = load_catalog(config_file) - runtime = resolve_runtime_settings(catalog) - _, updates, env_vars = resolve_model_profile( - content=catalog, - requested_model=runtime["model_key"], - requested_tp=runtime["tensor_parallel_size"], - ) - for key, value in env_vars.items(): - os.environ[key] = value - host = str(runtime["openai_host"]) - port = str(runtime["openai_port"]) - public_model_name = str(runtime["public_model_name"]).strip() - default_enable_thinking = bool(runtime["default_enable_thinking"]) - reasoning_enabled = bool(runtime["reasoning_enabled"]) - api_key = runtime["api_key"] or "" - dtype = str(updates["dtype"] or runtime["dtype"]) - quantization = str(updates["quantization"] or "").strip() - model_impl = str(updates["model_impl"] or "").strip() - reasoning_parser = str(updates["reasoning_parser"] or "").strip() - revision = runtime["revision"] or "" + config = load_app_config("config.json") + host = str(config["openai_host"]) + port = str(config["openai_port"]) + public_model_name = str(config["public_model_name"]).strip() + default_enable_thinking = bool(config["default_enable_thinking"]) + reasoning_enabled = bool(config["reasoning_enabled"]) + api_key = config.get("api_key") or "" + dtype = str(config.get("dtype", "bfloat16")) + quantization = str(config.get("quantization", "")).strip() + model_impl = str(config.get("model_impl", "")).strip() + reasoning_parser = str(config.get("reasoning_parser", "")).strip() + revision = config.get("revision", "") or "" cmd = [ sys.executable, "-m", @@ -37,28 +27,28 @@ def build_command() -> list[str]: "--port", port, "--model", - str(updates["model_name"]), + str(config["model_name"]), "--served-model-name", - public_model_name or str(updates["served_model_name"]), + public_model_name or str(config["served_model_name"]), "--tensor-parallel-size", - str(updates["tensor_parallel_size"]), + str(config["tensor_parallel_size"]), "--max-model-len", - str(updates["max_model_len"]), + str(config["max_model_len"]), "--gpu-memory-utilization", - str(updates["gpu_memory_utilization"]), + str(config["gpu_memory_utilization"]), "--max-num-seqs", - str(updates["max_num_seqs"]), + str(config["max_num_seqs"]), "--dtype", dtype, ] - if updates["trust_remote_code"]: + if config["trust_remote_code"]: cmd.append("--trust-remote-code") - if updates["enforce_eager"]: + if config["enforce_eager"]: cmd.append("--enforce-eager") - if updates["enable_auto_tool_choice"]: + if config["enable_auto_tool_choice"]: cmd.append("--enable-auto-tool-choice") - if updates["tool_call_parser"]: - cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])]) + if config["tool_call_parser"]: + cmd.extend(["--tool-call-parser", str(config["tool_call_parser"])]) cmd.extend( [ "--default-chat-template-kwargs", @@ -75,12 +65,12 @@ def build_command() -> list[str]: cmd.extend(["--revision", revision]) if api_key: cmd.extend(["--api-key", api_key]) - if updates.get("kv_cache_dtype"): - cmd.extend(["--kv-cache-dtype", str(updates["kv_cache_dtype"])]) - if updates.get("enable_prefix_caching"): + if config.get("kv_cache_dtype"): + cmd.extend(["--kv-cache-dtype", str(config["kv_cache_dtype"])]) + if config.get("enable_prefix_caching"): cmd.append("--enable-prefix-caching") - if updates.get("max_num_batched_tokens", 0) > 0: - cmd.extend(["--max-num-batched-tokens", str(updates["max_num_batched_tokens"])]) + if config.get("max_num_batched_tokens", 0) > 0: + cmd.extend(["--max-num-batched-tokens", str(config["max_num_batched_tokens"])]) if updates.get("language_model_only"): cmd.append("--language-model-only") speculative_method = str(updates.get("speculative_method") or "").strip() diff --git a/config.json b/config.json index 92ceb10..8509f7b 100644 --- a/config.json +++ b/config.json @@ -69,7 +69,7 @@ "local_path": "Qwen3.6-27B-FP8", "dtype": "auto", "quantization": "fp8", - "ctx": "65536", + "ctx": "131072", "max_tokens": "32768", "max_num_batched_tokens": 16384, "trust_remote": true, @@ -79,7 +79,7 @@ 2 ], "max_num_seqs": 32, - "gpu_util": "0.88", + "gpu_util": "0.75", "tool_call_parser": "qwen3_coder", "reasoning_parser": "qwen3", "enable_auto_tool_choice": true,