x

2026-03-28 13:54:48 +08:00
parent 7ea8cf48c3
commit b563d85519
2 changed files with 64 additions and 1 deletions
@@ -114,7 +114,16 @@ def launch_model(model_id, config, model_path, gpu_count):
    # Get configuration
    valid_tp = config.get("valid_tp", [1])
    max_tp = max(valid_tp) if valid_tp else 1
-    tp_size = min(gpu_count, max_tp)
+    
+    # Check for manual TP_SIZE override
+    tp_size_env = os.getenv("TP_SIZE")
+    if tp_size_env:
+        tp_size = int(tp_size_env)
+        log(f"TP_SIZE environment variable set: {tp_size}")
+        if tp_size not in valid_tp:
+            log(f"WARNING: TP_SIZE={tp_size} is not in valid_tp={valid_tp}, proceeding anyway")
+    else:
+        tp_size = min(gpu_count, max_tp)
    
    ctx = int(config.get("ctx", 8192))
    max_seqs = int(config.get("max_num_seqs", 64))