x

2026-03-11 00:02:45 +08:00
parent 65ccd278c9
commit a7d6e007e0
1 changed files with 110 additions and 0 deletions
@@ -0,0 +1,110 @@
+# model_config.yaml - 多模型配置示例
+# 默认启动的模型
+default: "deepseek_r1_distill_qwen_32b_awq"
+
+# 模型配置
+models:
+  deepseek_r1_distill_qwen_14b:
+    path: "/model/DeepSeek-R1-Distill-Qwen-14B"
+    name: "DeepSeek-R1-Distill-Qwen-14B"
+    max_model_len: 8192
+    gpu_memory_utilization: 0.9
+    port: 2001
+    dtype: "float16"
+    quantization: "awq"
+    tensor_parallel_size: 1
+    enforce_eager: true
+    api_key: "sk-14b-20240101-abcdef123456"
+    
+  deepseek_r1_distill_qwen_32b_awq:
+    path: "/model/DeepSeek-R1-Distill-Qwen-32B-AWQ"
+    name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"
+    
+    # 模型性能参数
+    max_model_len: 32768
+    gpu_memory_utilization: 0.95
+    enforce_eager: true
+    max_num_seqs: 2
+    max_num_batched_tokens: 1024
+    block_size: 16
+    tensor_parallel_size: 1
+    swap_space: 0
+    
+    # 新增：采样参数默认值
+    sampling_defaults:
+      temperature: 0.6
+      max_tokens: 4096
+      top_p: 0.9
+      frequency_penalty: 0.0
+      presence_penalty: 0.0
+      stop:
+        - "用户:"
+        - "助手:"
+        - "###"
+        - "问题:"
+        - "回答:"
+    
+    # 其他配置
+    dtype: "auto"
+    quantization: "awq"
+    port: 2001
+    api_key: "sk-32b-20240101-ghijk789012"
+    
+  glm_4_7_flash_awq:
+    path: "/model/GLM-4.7-Flash-AWQ"
+    name: "GLM-4.7-Flash-AWQ"
+
+    # 模型性能参数
+    max_model_len: 32768
+    gpu_memory_utilization: 0.9
+    enforce_eager: false
+    max_num_seqs: 2
+    max_num_batched_tokens: 1024
+    block_size: 16
+    tensor_parallel_size: 1
+    swap_space: 0
+    
+    # 新增：采样参数默认值
+    sampling_defaults:
+      temperature: 0.6
+      max_tokens: 4096
+      top_p: 0.9
+      frequency_penalty: 0.0
+      presence_penalty: 0.0
+      stop:
+        - "用户:"
+        - "助手:"
+        - "###"
+        - "问题:"
+        - "回答:"
+    
+    # 其他配置
+    dtype: "auto"
+    quantization: "awq"
+    port: 2001
+    api_key: "sk-32b-20240101-ghijk789012"
+
+  qwen3_vl_32b_instruct_awq:
+    path: "/model/Qwen3-VL-32B-Instruct-AWQ"
+    name: "Qwen3-VL-32B-Instruct-AWQ"
+    max_model_len: 32768
+    gpu_memory_utilization: 0.7
+    port: 2001
+    dtype: "auto"
+    quantization: "awq"
+    tensor_parallel_size: 1
+    enforce_eager: true
+    api_key: "sk-glm-20240101-lmnop345678"
+
+# 服务器通用设置
+server:
+  host: "0.0.0.0"
+  log_level: "info"
+  # 全局管理员密钥（拥有所有模型的访问权限）
+  admin_key: "sk-admin-20240101-xyz789"
+  # 允许的请求头名称（支持多个，按顺序检查）
+  api_key_headers: ["Authorization", "X-API-Key", "api-key"]
+  # 是否允许通过查询参数传递密钥
+  allow_query_param: true
+  # 查询参数名称
+  api_key_param: "api_key"