24 lines
554 B
Docker
24 lines
554 B
Docker
FROM rocm/vllm:rocm7.12.0_gfx120X-all_ubuntu24.04_py3.12_pytorch_2.9.1_vllm_0.16.0
|
|
|
|
WORKDIR /workspace
|
|
|
|
COPY requirements.txt /workspace/requirements.txt
|
|
RUN pip install --no-cache-dir -r /workspace/requirements.txt
|
|
|
|
COPY app /workspace/app
|
|
|
|
ENV PYTHONUNBUFFERED=1
|
|
ENV HOST=0.0.0.0
|
|
ENV PORT=8000
|
|
ENV TENSOR_PARALLEL_SIZE=2
|
|
ENV GPU_MEMORY_UTILIZATION=0.92
|
|
ENV MAX_MODEL_LEN=8192
|
|
ENV MAX_NUM_SEQS=64
|
|
ENV DTYPE=bfloat16
|
|
ENV ENFORCE_EAGER=false
|
|
ENV TRUST_REMOTE_CODE=false
|
|
|
|
EXPOSE 8000
|
|
|
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|