diff --git a/config.json b/config.json index d7959ff..c3ec432 100644 --- a/config.json +++ b/config.json @@ -78,7 +78,7 @@ "Qwen3.5-35B-A3B-FP8": { "local_path": "Qwen3.5-35B-A3B-FP8", "dtype": "float16", - "quantization": "None", + "quantization": "awq", "ctx": "131072", "enforce_eager": false, "trust_remote": true, diff --git a/docker-compose.yml b/docker-compose.yml index 49b4a4f..9934087 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,14 +19,14 @@ services: TRANSFORMERS_OFFLINE: "1" HF_DATASETS_OFFLINE: "1" # vLLM 相关配置 - VLLM_RPC_TIMEOUT: "300" - VLLM_WORKER_MULTIPROC_METHOD: "spawn" - PYTHONUNBUFFERED: "1" - # 启用 AITER 加速(R9700 属于 gfx1201 架构) - VLLM_ROCM_USE_AITER: "0" - VLLM_ROCM_USE_AITER_MHA: "0" - # 增强 All-Reduce 性能(双卡通信优化) - VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4" + # VLLM_RPC_TIMEOUT: "300" + # VLLM_WORKER_MULTIPROC_METHOD: "spawn" + # PYTHONUNBUFFERED: "1" + # # 启用 AITER 加速(R9700 属于 gfx1201 架构) + # VLLM_ROCM_USE_AITER: "0" + # VLLM_ROCM_USE_AITER_MHA: "0" + # # 增强 All-Reduce 性能(双卡通信优化) + # VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4" devices: - /dev/kfd - /dev/dri