x

Dockerfile

@@ -7,13 +7,17 @@ ENV LOCAL_MODEL_DIR=/opt/model
 # Create necessary directories
 RUN mkdir -p /opt/script /opt/model /config
 
-# Update transformers to support qwen3_5_moe architecture
-RUN pip install --upgrade transformers
-
 # Copy scripts to /opt/script
+COPY scripts/add_qwen3_5_moe_support.py /opt/script/add_qwen3_5_moe_support.py
 COPY scripts/start_vllm.py /opt/script/start-vllm
 COPY benchmarks/run_vllm_bench.py /opt/script/run_vllm_bench.py
 
+# Update vLLM to nightly version for Qwen3.5 support
+RUN pip install --upgrade vllm --extra-index-url https://wheels.vllm.ai/nightly
+
+# Add qwen3_5_moe support to Transformers
+RUN python3 /opt/script/add_qwen3_5_moe_support.py
+
 # Note: config.json should be mounted at runtime via docker-compose.yml
 
 # Make scripts executable

scripts/add_qwen3_5_moe_support.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""
+Add qwen3_5_moe architecture support to Transformers
+"""
+import os
+import sys
+
+def add_qwen3_5_moe_support():
+    """Add qwen3_5_moe to Transformers configuration"""
+    
+    # Find transformers installation path
+    try:
+        import transformers
+        transformers_path = transformers.__path__[0]
+    except ImportError:
+        print("Transformers not installed")
+        return False
+    
+    print(f"Transformers path: {transformers_path}")
+    
+    # Modify configuration_auto.py
+    config_auto_file = os.path.join(transformers_path, "models", "auto", "configuration_auto.py")
+    
+    if not os.path.exists(config_auto_file):
+        print(f"Config file not found: {config_auto_file}")
+        return False
+    
+    with open(config_auto_file, "r") as f:
+        content = f.read()
+    
+    # Check if already exists
+    if "qwen3_5_moe" in content:
+        print("qwen3_5_moe already exists in configuration")
+        return True
+    
+    # Add to MODEL_NAMES_MAPPING
+    # Find the line with "qwen2_moe": "Qwen2Moe" and add qwen3_5_moe after it
+    if '"qwen2_moe": "Qwen2Moe"' in content:
+        content = content.replace(
+            '"qwen2_moe": "Qwen2Moe"',
+            '"qwen2_moe": "Qwen2Moe",\n        "qwen3_5_moe": "Qwen3_5_MoE"'
+        )
+        print("Added qwen3_5_moe to MODEL_NAMES_MAPPING")
+    
+    # Add to MODEL_MAPPING (mapping to Qwen2MoeConfig as base)
+    if '"qwen2_moe": Qwen2MoeConfig' in content:
+        content = content.replace(
+            '"qwen2_moe": Qwen2MoeConfig',
+            '"qwen2_moe": Qwen2MoeConfig,\n        "qwen3_5_moe": Qwen2MoeConfig'
+        )
+        print("Added qwen3_5_moe to MODEL_MAPPING")
+    
+    with open(config_auto_file, "w") as f:
+        f.write(content)
+    
+    print("Successfully added qwen3_5_moe support")
+    return True
+
+if __name__ == "__main__":
+    success = add_qwen3_5_moe_support()
+    sys.exit(0 if success else 1)

scripts/start_vllm.py

@@ -140,6 +140,12 @@ def launch_model(model_id, config, model_path, gpu_count):
     if config.get("enforce_eager"):
         cmd.append("--enforce-eager")
     
+    # Add Qwen3.5 specific parameters
+    if "qwen3.5" in model_id.lower() or "qwen3_5" in model_id.lower():
+        cmd.extend(["--quantization", "moe_wna16"])
+        cmd.extend(["--reasoning-parser", "qwen3"])
+        log("Added Qwen3.5 specific parameters: --quantization moe_wna16 --reasoning-parser qwen3")
+    
     log(f"Command: {' '.join(cmd)}")
     
     # Set environment