270 lines
6.0 KiB
YAML
270 lines
6.0 KiB
YAML
# vLLM 参数配置文件
|
|
# 用于定义和识别 vLLM 服务器的各类参数
|
|
|
|
# 模型参数配置
|
|
model_params:
|
|
# 必需参数
|
|
required:
|
|
- path # 模型路径
|
|
|
|
# 可选参数
|
|
optional:
|
|
- name # 模型名称
|
|
- dtype # 数据类型 (auto, float16, float32, bfloat16)
|
|
- quantization # 量化方式 (awq, gptq, squeezellm)
|
|
- trust_remote # 是否信任远程代码
|
|
|
|
# 性能参数配置
|
|
performance_params:
|
|
# GPU 相关
|
|
gpu:
|
|
- tensor_parallel_size # 张量并行度
|
|
- pipeline_parallel_size # 流水线并行度
|
|
- gpu_memory_utilization # GPU 内存利用率 (0.0-1.0)
|
|
- swap_space # CPU 交换空间大小 (GB)
|
|
- max_num_batched_tokens # 最大批量 tokens 数
|
|
- max_num_seqs # 最大并发请求数
|
|
- num_scheduler_steps # 调度器步数
|
|
|
|
# 内存管理
|
|
memory:
|
|
- block_size # 块大小 (8, 16, 32)
|
|
- max_model_len # 最大模型上下文长度
|
|
- max_logprobs # 最大 logprobs 数
|
|
- disable_sliding_window # 禁用滑动窗口
|
|
|
|
# 服务参数配置
|
|
server_params:
|
|
# 网络配置
|
|
network:
|
|
- host # 服务器主机地址
|
|
- port # 服务器端口
|
|
- ssl_keyfile # SSL 密钥文件
|
|
- ssl_certfile # SSL 证书文件
|
|
- ssl_ca_certs # SSL CA 证书
|
|
- ssl_keyfile_password # SSL 密钥密码
|
|
|
|
# API 配置
|
|
api:
|
|
- api_key # API 密钥
|
|
- allowed_origins # 允许的源
|
|
- timeout_keep_alive # 保持连接超时时间
|
|
|
|
# 采样参数配置
|
|
sampling_params:
|
|
# 温度控制
|
|
- temperature # 温度 (0.0-2.0)
|
|
- top_p # 核采样参数 (0.0-1.0)
|
|
- top_k # Top-K 采样
|
|
- min_p # 最小概率
|
|
|
|
# 惩罚参数
|
|
- frequency_penalty # 频率惩罚 (-2.0 到 2.0)
|
|
- presence_penalty # 存在惩罚 (-2.0 到 2.0)
|
|
- repetition_penalty # 重复惩罚
|
|
|
|
# 生成长度
|
|
- max_tokens # 最大生成 tokens 数
|
|
- min_tokens # 最小生成 tokens 数
|
|
- stop # 停止词列表
|
|
- stop_token_ids # 停止 token IDs
|
|
|
|
# 其他采样选项
|
|
- seed # 随机种子
|
|
- use_beam_search # 使用束搜索
|
|
- best_of # 束搜索的最佳候选数
|
|
- length_penalty # 长度惩罚
|
|
- early_stopping # 早期停止
|
|
- ignore_eos # 忽略 EOS token
|
|
- skip_special_tokens # 跳过特殊 tokens
|
|
- spaces_between_special_tokens # 特殊 tokens 之间的空格
|
|
|
|
# 日志参数配置
|
|
logging_params:
|
|
- log_level # 日志级别 (debug, info, warning, error)
|
|
- log_requests # 是否记录请求
|
|
- log_responses # 是否记录响应
|
|
|
|
# 高级参数配置
|
|
advanced_params:
|
|
# 执行模式
|
|
- enforce_eager # 强制使用 eager 模式
|
|
- cuda_graphs # CUDA 图
|
|
- use_v2_block_manager # 使用 V2 块管理器
|
|
|
|
# 注意力后端
|
|
- use_rocm_attn # 使用 ROCm 注意力后端
|
|
- attention_backend # 注意力后端类型
|
|
|
|
# 分布式
|
|
- distributed_executor_backend # 分布式执行器后端
|
|
- ray_workers_use_nsight # Ray workers 使用 nsight
|
|
|
|
# 其他
|
|
- load_format # 加载格式
|
|
- download_dir # 下载目录
|
|
- revision # 模型版本
|
|
- code_revision # 代码版本
|
|
- tokenizer_revision # tokenizer 版本
|
|
|
|
# 参数类型映射
|
|
param_types:
|
|
# 整数类型
|
|
integer:
|
|
- tensor_parallel_size
|
|
- pipeline_parallel_size
|
|
- max_num_batched_tokens
|
|
- max_num_seqs
|
|
- block_size
|
|
- max_model_len
|
|
- max_tokens
|
|
- min_tokens
|
|
- top_k
|
|
- best_of
|
|
- seed
|
|
- num_scheduler_steps
|
|
- max_logprobs
|
|
- swap_space
|
|
- timeout_keep_alive
|
|
|
|
# 浮点数类型
|
|
float:
|
|
- gpu_memory_utilization
|
|
- temperature
|
|
- top_p
|
|
- min_p
|
|
- frequency_penalty
|
|
- presence_penalty
|
|
- repetition_penalty
|
|
- length_penalty
|
|
|
|
# 布尔类型
|
|
boolean:
|
|
- trust_remote
|
|
- enforce_eager
|
|
- use_rocm_attn
|
|
- use_beam_search
|
|
- early_stopping
|
|
- ignore_eos
|
|
- skip_special_tokens
|
|
- spaces_between_special_tokens
|
|
- log_requests
|
|
- log_responses
|
|
- disable_sliding_window
|
|
- use_v2_block_manager
|
|
|
|
# 字符串类型
|
|
string:
|
|
- path
|
|
- name
|
|
- dtype
|
|
- quantization
|
|
- host
|
|
- port
|
|
- api_key
|
|
- ssl_keyfile
|
|
- ssl_certfile
|
|
- ssl_ca_certs
|
|
- ssl_keyfile_password
|
|
- log_level
|
|
- attention_backend
|
|
- load_format
|
|
- download_dir
|
|
- revision
|
|
- code_revision
|
|
- tokenizer_revision
|
|
|
|
# 列表类型
|
|
list:
|
|
- stop
|
|
- stop_token_ids
|
|
- allowed_origins
|
|
|
|
# 参数默认值
|
|
param_defaults:
|
|
host: "0.0.0.0"
|
|
port: 8000
|
|
dtype: "auto"
|
|
gpu_memory_utilization: 0.9
|
|
max_model_len: 8192
|
|
tensor_parallel_size: 1
|
|
max_num_seqs: 256
|
|
block_size: 16
|
|
temperature: 0.7
|
|
top_p: 0.9
|
|
max_tokens: 256
|
|
enforce_eager: false
|
|
trust_remote: false
|
|
log_level: "info"
|
|
|
|
# 参数验证规则
|
|
param_validation:
|
|
gpu_memory_utilization:
|
|
min: 0.0
|
|
max: 1.0
|
|
|
|
temperature:
|
|
min: 0.0
|
|
max: 2.0
|
|
|
|
top_p:
|
|
min: 0.0
|
|
max: 1.0
|
|
|
|
min_p:
|
|
min: 0.0
|
|
max: 1.0
|
|
|
|
frequency_penalty:
|
|
min: -2.0
|
|
max: 2.0
|
|
|
|
presence_penalty:
|
|
min: -2.0
|
|
max: 2.0
|
|
|
|
tensor_parallel_size:
|
|
min: 1
|
|
max: 8
|
|
|
|
block_size:
|
|
allowed_values: [8, 16, 32]
|
|
|
|
# 参数分组（用于配置文件组织）
|
|
param_groups:
|
|
basic:
|
|
name: "基础配置"
|
|
params:
|
|
- path
|
|
- name
|
|
- port
|
|
- api_key
|
|
|
|
performance:
|
|
name: "性能配置"
|
|
params:
|
|
- tensor_parallel_size
|
|
- gpu_memory_utilization
|
|
- max_model_len
|
|
- max_num_seqs
|
|
- block_size
|
|
|
|
sampling:
|
|
name: "采样配置"
|
|
params:
|
|
- temperature
|
|
- top_p
|
|
- max_tokens
|
|
- frequency_penalty
|
|
- presence_penalty
|
|
- stop
|
|
|
|
advanced:
|
|
name: "高级配置"
|
|
params:
|
|
- dtype
|
|
- quantization
|
|
- enforce_eager
|
|
- trust_remote
|
|
- swap_space
|