x

2026-03-29 05:06:19 +08:00
parent 67eb27f2d2
commit 617665c6ab
12 changed files with 452 additions and 1 deletions
@@ -0,0 +1,23 @@
+FROM rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0
+
+WORKDIR /workspace
+
+COPY requirements.txt /workspace/requirements.txt
+RUN pip install --no-cache-dir -r /workspace/requirements.txt
+
+COPY app /workspace/app
+
+ENV PYTHONUNBUFFERED=1
+ENV HOST=0.0.0.0
+ENV PORT=8000
+ENV TENSOR_PARALLEL_SIZE=2
+ENV GPU_MEMORY_UTILIZATION=0.92
+ENV MAX_MODEL_LEN=8192
+ENV MAX_NUM_SEQS=64
+ENV DTYPE=bfloat16
+ENV ENFORCE_EAGER=false
+ENV TRUST_REMOTE_CODE=false
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]