113 lines
2.7 KiB
YAML
113 lines
2.7 KiB
YAML
# model_config.yaml - 多模型配置示例
|
|
# 默认启动的模型
|
|
default: "deepseek_r1_distill_qwen_32b_awq"
|
|
|
|
|
|
|
|
# 模型配置
|
|
models:
|
|
deepseek_r1_distill_qwen_14b:
|
|
path: "/model/DeepSeek-R1-Distill-Qwen-14B"
|
|
name: "DeepSeek-R1-Distill-Qwen-14B"
|
|
max_model_len: 8192
|
|
gpu_memory_utilization: 0.9
|
|
port: 2001
|
|
dtype: "float16"
|
|
quantization: "awq"
|
|
tensor_parallel_size: 1
|
|
enforce_eager: true
|
|
api_key: "sk-14b-20240101-abcdef123456"
|
|
|
|
deepseek_r1_distill_qwen_32b_awq:
|
|
path: "/model/DeepSeek-R1-Distill-Qwen-32B-AWQ"
|
|
name: "DeepSeek-R1-Distill-Qwen-32B-AWQ"
|
|
|
|
# 模型性能参数
|
|
max_model_len: 32768
|
|
gpu_memory_utilization: 0.95
|
|
enforce_eager: true
|
|
max_num_seqs: 2
|
|
max_num_batched_tokens: 1024
|
|
block_size: 16
|
|
tensor_parallel_size: 1
|
|
swap_space: 0
|
|
|
|
# 新增：采样参数默认值
|
|
sampling_defaults:
|
|
temperature: 0.6
|
|
max_tokens: 4096
|
|
top_p: 0.9
|
|
frequency_penalty: 0.0
|
|
presence_penalty: 0.0
|
|
stop:
|
|
- "用户:"
|
|
- "助手:"
|
|
- "###"
|
|
- "问题:"
|
|
- "回答:"
|
|
|
|
# 其他配置
|
|
dtype: "auto"
|
|
quantization: "awq"
|
|
port: 2001
|
|
api_key: "sk-32b-20240101-ghijk789012"
|
|
|
|
glm_4_7_flash_awq:
|
|
path: "/model/GLM-4.7-Flash-AWQ"
|
|
name: "GLM-4.7-Flash-AWQ"
|
|
|
|
# 模型性能参数
|
|
max_model_len: 32768
|
|
gpu_memory_utilization: 0.9
|
|
enforce_eager: false
|
|
max_num_seqs: 2
|
|
max_num_batched_tokens: 1024
|
|
block_size: 16
|
|
tensor_parallel_size: 1
|
|
swap_space: 0
|
|
|
|
# 新增：采样参数默认值
|
|
sampling_defaults:
|
|
temperature: 0.6
|
|
max_tokens: 4096
|
|
top_p: 0.9
|
|
frequency_penalty: 0.0
|
|
presence_penalty: 0.0
|
|
stop:
|
|
- "用户:"
|
|
- "助手:"
|
|
- "###"
|
|
- "问题:"
|
|
- "回答:"
|
|
|
|
# 其他配置
|
|
dtype: "auto"
|
|
quantization: "awq"
|
|
port: 2001
|
|
api_key: "sk-32b-20240101-ghijk789012"
|
|
|
|
qwen3_vl_32b_instruct_awq:
|
|
path: "/model/Qwen3-VL-32B-Instruct-AWQ"
|
|
name: "Qwen3-VL-32B-Instruct-AWQ"
|
|
max_model_len: 32768
|
|
gpu_memory_utilization: 0.7
|
|
port: 2001
|
|
dtype: "auto"
|
|
quantization: "awq"
|
|
tensor_parallel_size: 1
|
|
enforce_eager: true
|
|
api_key: "sk-glm-20240101-lmnop345678"
|
|
|
|
# 服务器通用设置
|
|
server:
|
|
host: "0.0.0.0"
|
|
log_level: "info"
|
|
# 全局管理员密钥（拥有所有模型的访问权限）
|
|
admin_key: "sk-admin-20240101-xyz789"
|
|
# 允许的请求头名称（支持多个，按顺序检查）
|
|
api_key_headers: ["Authorization", "X-API-Key", "api-key"]
|
|
# 是否允许通过查询参数传递密钥
|
|
allow_query_param: true
|
|
# 查询参数名称
|
|
api_key_param: "api_key"
|