x

docker-compose.yml

@@ -6,8 +6,15 @@ services:
     environment:
       - USE_DEFAULT_MODEL=true
       - LOCAL_MODEL_DIR=/opt/model
+      - HIP_VISIBLE_DEVICES=0,1
+      - NCCL_DEBUG=INFO
+      - NCCL_SOCKET_IFNAME=^lo,docker0
+      - NCCL_P2P_DISABLE=0
+      - NCCL_SHM_DISABLE=0
+      - NCCL_IB_DISABLE=1
+      - NCCL_P2P_LEVEL=SYS
     volumes:
-      - /opt/project/amd-r9700-vllm-toolboxes/config.json:/config/config.json:ro
+      - ./config.json:/config/config.json:ro
       - /opt/model:/opt/model
     devices:
       - /dev/dri:/dev/dri

scripts/start_vllm.py

@@ -161,6 +161,18 @@ def launch_model(model_id, config, model_path, gpu_count):
     env = os.environ.copy()
     env.update(config.get("env", {}))
     
+    # Add AMD GPU tensor parallel environment variables
+    if tp_size > 1:
+        env["NCCL_DEBUG"] = "INFO"
+        env["NCCL_SOCKET_IFNAME"] = "^lo,docker0"
+        env["NCCL_P2P_DISABLE"] = "0"
+        env["NCCL_SHM_DISABLE"] = "0"
+        env["NCCL_IB_DISABLE"] = "1"
+        env["NCCL_P2P_LEVEL"] = "SYS"
+        env["HIP_VISIBLE_DEVICES"] = os.getenv("HIP_VISIBLE_DEVICES", "0,1")
+        log("Added NCCL environment variables for AMD GPU tensor parallel")
+        log(f"HIP_VISIBLE_DEVICES: {env['HIP_VISIBLE_DEVICES']}")
+    
     # Launch vLLM
     log("Starting vLLM server...")
     try: