x

app/start_openai.py

@@ -19,7 +19,8 @@ def build_command() -> list[str]:
     host = str(runtime["openai_host"])
     port = str(runtime["openai_port"])
     api_key = runtime["api_key"] or ""
-    dtype = str(runtime["dtype"])
+    dtype = str(updates["dtype"] or runtime["dtype"])
+    quantization = str(updates["quantization"] or "").strip()
     revision = runtime["revision"] or ""
     cmd = [
         sys.executable,
@@ -52,6 +53,8 @@ def build_command() -> list[str]:
         cmd.append("--enable-auto-tool-choice")
     if updates["tool_call_parser"]:
         cmd.extend(["--tool-call-parser", str(updates["tool_call_parser"])])
+    if quantization:
+        cmd.extend(["--quantization", quantization])
     if revision:
         cmd.extend(["--revision", revision])
     if api_key: