x

config.json

@@ -78,7 +78,7 @@
       "Qwen3.5-35B-A3B-FP8": {
         "local_path": "Qwen3.5-35B-A3B-FP8",
         "dtype": "float16",
-        "quantization": "None",
+        "quantization": "awq",
         "ctx": "131072",
         "enforce_eager": false,
         "trust_remote": true,

docker-compose.yml

@@ -19,14 +19,14 @@ services:
       TRANSFORMERS_OFFLINE: "1"
       HF_DATASETS_OFFLINE: "1"
       # vLLM 相关配置
-      VLLM_RPC_TIMEOUT: "300"
-      VLLM_WORKER_MULTIPROC_METHOD: "spawn"
-      PYTHONUNBUFFERED: "1"
-      # 启用 AITER 加速（R9700 属于 gfx1201 架构）
-      VLLM_ROCM_USE_AITER: "0"
-      VLLM_ROCM_USE_AITER_MHA: "0"
-      # 增强 All-Reduce 性能（双卡通信优化）
-      VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4"
+      # VLLM_RPC_TIMEOUT: "300"
+      # VLLM_WORKER_MULTIPROC_METHOD: "spawn"
+      # PYTHONUNBUFFERED: "1"
+      # # 启用 AITER 加速（R9700 属于 gfx1201 架构）
+      # VLLM_ROCM_USE_AITER: "0"
+      # VLLM_ROCM_USE_AITER_MHA: "0"
+      # # 增强 All-Reduce 性能（双卡通信优化）
+      # VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: "INT4"
     devices:
       - /dev/kfd
       - /dev/dri