diff --git a/README.md b/README.md index 3c242c26..2f98b9bc 100644 --- a/README.md +++ b/README.md @@ -127,10 +127,10 @@ client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/b ```bash docker run -p 9090:9090 --runtime=nvidia --gpus all --entrypoint /bin/bash -it ghcr.io/collabora/whisperlive-tensorrt - # Build tiny.en engine + # Build small.en engine bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en - # Run server with tiny.en + # Run server with small.en python3 run_server.py --port 9090 \ --backend tensorrt \ --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_en" diff --git a/docker/Dockerfile.tensorrt b/docker/Dockerfile.tensorrt index de6027ef..3607ab3c 100644 --- a/docker/Dockerfile.tensorrt +++ b/docker/Dockerfile.tensorrt @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.1.0-runtime-ubuntu22.04 +FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04 AS base ARG DEBIAN_FRONTEND=noninteractive @@ -6,14 +6,16 @@ RUN apt-get update && apt-get install -y \ python3.10 python3-pip openmpi-bin libopenmpi-dev git wget \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install --no-cache-dir -U tensorrt_llm==0.9.0 --extra-index-url https://pypi.nvidia.com - +FROM base AS devel +RUN pip3 install --no-cache-dir -U tensorrt_llm==0.10.0 --extra-index-url https://pypi.nvidia.com WORKDIR /app - -RUN git clone -b v0.9.0 --depth 1 https://github.com/NVIDIA/TensorRT-LLM.git && \ +RUN git clone -b v0.10.0 --depth 1 https://github.com/NVIDIA/TensorRT-LLM.git && \ mv TensorRT-LLM/examples ./TensorRT-LLM-examples && \ rm -rf TensorRT-LLM + +FROM devel AS release +WORKDIR /app COPY assets/ ./assets RUN wget -nc -P assets/ https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz @@ -22,7 +24,7 @@ RUN apt update && bash setup.sh && rm setup.sh COPY requirements/server.txt . RUN pip install --no-cache-dir -r server.txt && rm server.txt - +RUN pip install -U huggingface_hub tokenizers==0.19.0 COPY whisper_live ./whisper_live COPY scripts/build_whisper_tensorrt.sh . COPY run_server.py . \ No newline at end of file diff --git a/requirements/server.txt b/requirements/server.txt index 6f5132fd..f1f873c8 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -1,13 +1,13 @@ faster-whisper==1.0.1 -torch==2.3.0 websockets onnxruntime==1.16.0 numba -openai-whisper kaldialign soundfile ffmpeg-python scipy jiwer evaluate -numpy<2 \ No newline at end of file +numpy<2 +tiktoken==0.3.3 +openai-whisper \ No newline at end of file