From 6eb8a7c24acb434a2cb88cefa41e5434c1122889 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Thu, 13 Jun 2024 18:18:35 -0700 Subject: [PATCH 01/19] Refactor the build to build everything in docker containers so it can be platform agnostic Update .gitignore --- .gitignore | 2 + Makefile | 94 +++++++++++++--------------- packages/api/Dockerfile | 27 ++++---- packages/llama-cpp-python/Dockerfile | 26 +++++--- packages/text-embeddings/Dockerfile | 20 +++--- packages/vllm/Dockerfile | 15 +++-- packages/whisper/Dockerfile | 21 ++++--- src/leapfrogai_sdk/Dockerfile | 13 ++++ 8 files changed, 126 insertions(+), 92 deletions(-) create mode 100644 src/leapfrogai_sdk/Dockerfile diff --git a/.gitignore b/.gitignore index 5e0f60a7d..b7c0db10f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ build/ .branches .temp src/leapfrogai_api/config.yaml +.idea/ +/.python-version # local model and tokenizer files *.bin diff --git a/Makefile b/Makefile index 4cd55db46..377185eb2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ ARCH ?= amd64 KEY ?= "" +REG_PORT ?= 5000 VERSION ?= $(shell git describe --abbrev=0 --tags) LOCAL_VERSION ?= $(shell git rev-parse --short HEAD) @@ -31,123 +32,112 @@ gen-python: ## Generate the protobufs for the OpenAI typing within the leapfroga src/leapfrogai_sdk/proto/leapfrogai_sdk/**/*.proto local-registry: ## Start up a local container registry. Errors in this target are ignored. - -docker run -d -p 5000:5000 --restart=always --name registry registry:2 + -docker run -d -p ${REG_PORT}:5000 --restart=always --name registry registry:2 sdk-wheel: ## build wheels for the leapfrogai_sdk package as a dependency for other lfai components - -rm ${SDK_DEST}/*.whl - python -m pip wheel src/leapfrogai_sdk -w ${SDK_DEST} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest -f src/leapfrogai_sdk/Dockerfile . build-supabase: local-registry ## Build the migration container for this version of the supabase package docker build -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" . - docker tag ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} - docker push localhost:5000/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} + docker tag ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/supabase -o packages/supabase --registry-override=ghcr.io=localhost:5000 --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/supabase -o packages/supabase --registry-override=ghcr.io=localhost:${REG_PORT} --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-api-deps: sdk-wheel ## Download the wheels for the leapfrogai_api dependencies - -rm packages/api/build/*.whl - python -m pip wheel src/leapfrogai_api -w packages/api/build --find-links=${SDK_DEST} build-api: local-registry setup-api-deps ## Build the leapfrogai_api container and Zarf package ## Build the API image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} packages/api - docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . + docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} ## Build the migration container for this version of the API - docker build -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . - docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . + docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} ## Push the images to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} - docker push localhost:5000/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/api -o packages/api --registry-override=ghcr.io=localhost:5000 --insecure --set LEAPFROGAI_IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/api -o packages/api --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set LEAPFROGAI_IMAGE_VERSION=${LOCAL_VERSION} --confirm build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package ## Build the UI image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} src/leapfrogai_ui - docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} src/leapfrogai_ui + docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} ## Build the migration container for the version of the UI - docker build -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" . - docker tag ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" . + docker tag ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} - docker push localhost:5000/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/ui -o packages/ui --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/ui -o packages/ui --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-llama-cpp-python-deps: sdk-wheel ## Download the wheels for the optional 'llama-cpp-python' dependencies - -rm packages/llama-cpp-python/build/*.whl - - ## The external link is needed to pull a pre-compiled cpu wheel for llama-cpp-python - python -m pip wheel packages/llama-cpp-python -w packages/llama-cpp-python/build --find-links=${SDK_DEST} build-llama-cpp-python: local-registry setup-llama-cpp-python-deps ## Build the llama-cpp-python (cpu) container and Zarf package ## Build the image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} packages/llama-cpp-python - docker tag ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} -f packages/llama-cpp-python/Dockerfile . + docker tag ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/llama-cpp-python -o packages/llama-cpp-python --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/llama-cpp-python -o packages/llama-cpp-python --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-vllm-deps: sdk-wheel ## Download the wheels for the optional 'vllm' dependencies - -rm packages/vllm/build/*.whl - python -m pip wheel packages/vllm -w packages/vllm/build --find-links=${SDK_DEST} + build-vllm: local-registry setup-vllm-deps ## Build the vllm container and Zarf package ## Build the image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} packages/vllm - docker tag ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} -f packages/vllm/Dockerfile . + docker tag ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/vllm -o packages/vllm --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/vllm -o packages/vllm --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-text-embeddings-deps: sdk-wheel ## Download the wheels for the optional 'text-embeddings' dependencies - -rm packages/text-embeddings/build/*.whl - python -m pip wheel packages/text-embeddings -w packages/text-embeddings/build --find-links=${SDK_DEST} + build-text-embeddings: local-registry setup-text-embeddings-deps ## Build the text-embeddings container and Zarf package ## Build the image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} packages/text-embeddings - docker tag ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} -f packages/text-embeddings/Dockerfile . + docker tag ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/text-embeddings -o packages/text-embeddings --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/text-embeddings -o packages/text-embeddings --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-whisper-deps: sdk-wheel ## Download the wheels for the optional 'whisper' dependencies - -rm packages/whisper/build/*.whl - python -m pip wheel "packages/whisper[dev]" -w packages/whisper/build --find-links=${SDK_DEST} build-whisper: local-registry setup-whisper-deps ## Build the whisper container and zarf package ## Build the image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} packages/whisper - docker tag ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} -f packages/whisper/Dockerfile . + docker tag ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/whisper -o packages/whisper --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/whisper -o packages/whisper --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-repeater-deps: sdk-wheel ## Download the wheels for the optional 'repeater' dependencies -rm packages/repeater/build/*.whl @@ -155,14 +145,14 @@ setup-repeater-deps: sdk-wheel ## Download the wheels for the optional 'repeater build-repeater: local-registry setup-repeater-deps ## Build the repeater container and zarf package ## Build the image (and tag it for the local registry) - docker build -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} packages/repeater - docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:5000/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} packages/repeater + docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) - docker push localhost:5000/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} + docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/repeater -o packages/repeater --registry-override=ghcr.io=localhost:5000 --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/repeater -o packages/repeater --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm build-cpu: build-supabase build-api build-ui build-llama-cpp-python build-text-embeddings build-whisper ## Build all zarf packages for a cpu-enabled deployment of LFAI diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index ccbad34b5..99559cc48 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -1,24 +1,29 @@ -ARG ARCH=amd64 +ARG ARCH +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder - +ARG SDK_DEST=src/leapfrogai_sdk/build +USER root WORKDIR /leapfrogai # copy the api dependencies over # NOTE: We are copying to this filename because installing 'optional extras' from a wheel requires the absolute path to the wheel file (instead of a wildcard whl) -COPY build/*.whl build/ -COPY build/leapfrogai_api*.whl . -RUN pip install leapfrogai_api*.whl --no-index --find-links=build/ - -FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-${ARCH} +COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} +COPY src/leapfrogai_api src/leapfrogai_api -WORKDIR /leapfrogai +RUN python -m venv .venv +ENV PATH="/leapfrogai/.venv/bin:$PATH" -COPY --from=builder /home/nonroot/.local/lib/python3.11/site-packages /home/nonroot/.local/lib/python3.11/site-packages -COPY --from=builder /home/nonroot/.local/bin/uvicorn /home/nonroot/.local/bin/uvicorn +RUN rm -f packages/api/build/*.whl +RUN python -m pip wheel src/leapfrogai_api -w packages/api/build --find-links=${SDK_DEST} +RUN pip install packages/api/build/leapfrogai_api*.whl --no-index --find-links=packages/api/build/ +FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-${ARCH} +ENV PATH="/leapfrogai/.venv/bin:$PATH" +WORKDIR /leapfrogai +COPY --from=builder /leapfrogai/.venv/ /leapfrogai/.venv/ EXPOSE 8080 -ENTRYPOINT ["/home/nonroot/.local/bin/uvicorn", "leapfrogai_api.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8080", "--log-level", "debug"] +ENTRYPOINT ["uvicorn", "leapfrogai_api.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8080", "--log-level", "debug"] diff --git a/packages/llama-cpp-python/Dockerfile b/packages/llama-cpp-python/Dockerfile index 6572d1809..ced587646 100644 --- a/packages/llama-cpp-python/Dockerfile +++ b/packages/llama-cpp-python/Dockerfile @@ -1,7 +1,10 @@ +ARG ARCH +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk # hardened and slim python w/ developer tools image FROM --platform=$BUILDPLATFORM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev as builder - +ARG SDK_DEST=src/leapfrogai_sdk/build +USER root WORKDIR /leapfrogai # download model @@ -12,10 +15,9 @@ ARG REVISION=3f65d882253d1f15a113dabf473a7c02a004d2b5 # NOTE: This is checking for a pre-downloaded model file in the local build dir before downloading the model from HuggingFace # TODO: Add checksum validation to verify the model in the local build-dir is the model we expect -COPY scripts/model_download.py scripts/model_download.py -COPY build/*.gguf .model/ -RUN if [[ -f .model/model.gguf ]] ; then : ; else REPO_ID=${REPO_ID} FILENAME=${FILENAME} REVISION=${REVISION} python3.11 scripts/model_download.py ; fi -RUN if [[ -f .model/model.gguf ]] ; then : ; else mv .model/*.gguf .model/model.gguf ; fi +COPY packages/llama-cpp-python/scripts/model_download.py scripts/model_download.py +RUN REPO_ID=${REPO_ID} FILENAME=${FILENAME} REVISION=${REVISION} python3.11 scripts/model_download.py +RUN mv .model/*.gguf .model/model.gguf # create virtual environment for light-weight portability and minimal libraries @@ -24,9 +26,13 @@ ENV PATH="/leapfrogai/.venv/bin:$PATH" # copy the llama-cpp-python build dependencies over # NOTE: We are copying to this filename because installing 'optional extras' from a wheel requires the absolute path to the wheel file (instead of a wildcard whl) -COPY build/*.whl build/ -COPY build/lfai_llama_cpp_python*.whl . -RUN pip install lfai_llama_cpp_python*.whl --no-index --find-links=build/ +COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} +COPY packages/llama-cpp-python packages/llama-cpp-python + +RUN rm -f packages/llama-cpp-python/build/*.whl +RUN python -m pip wheel packages/llama-cpp-python -w packages/llama-cpp-python/build --find-links=${SDK_DEST} + +RUN pip install packages/llama-cpp-python/build/lfai_llama_cpp_python*.whl --no-index --find-links=packages/llama-cpp-python/build/ # hardened and slim python image FROM --platform=$BUILDPLATFORM ghcr.io/defenseunicorns/leapfrogai/python:3.11 @@ -38,8 +44,8 @@ WORKDIR /leapfrogai COPY --from=builder /leapfrogai/.venv/ /leapfrogai/.venv/ COPY --from=builder /leapfrogai/.model/ /leapfrogai/.model/ -COPY main.py . -COPY config.yaml . +COPY packages/llama-cpp-python/main.py . +COPY packages/llama-cpp-python/config.yaml . EXPOSE 50051:50051 diff --git a/packages/text-embeddings/Dockerfile b/packages/text-embeddings/Dockerfile index 28fa67c24..23371b67c 100644 --- a/packages/text-embeddings/Dockerfile +++ b/packages/text-embeddings/Dockerfile @@ -1,8 +1,10 @@ -ARG ARCH=amd64 +ARG ARCH +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk # hardened and slim python w/ developer tools image FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder - +ARG SDK_DEST=src/leapfrogai_sdk/build +USER root WORKDIR /leapfrogai # create virtual environment for light-weight portability and minimal libraries @@ -12,16 +14,20 @@ ENV PATH="/leapfrogai/.venv/bin:$PATH" # copy and install all python dependencies # NOTE: We are copying the leapfrog whl to this filename because installing 'optional extras' from # a wheel requires the absolute path to the wheel file (instead of a wildcard whl) -COPY build/*.whl build/ -COPY build/lfai_text_embeddings*.whl . -RUN pip install lfai_text_embeddings*.whl --no-index --find-links=build/ + +COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} +COPY packages/text-embeddings packages/text-embeddings + +RUN rm -f packages/text-embeddings/build/*.whl +RUN python -m pip wheel packages/text-embeddings -w packages/text-embeddings/build --find-links=${SDK_DEST} +RUN pip install packages/text-embeddings/build/lfai_text_embeddings*.whl --no-index --find-links=packages/text-embeddings/build/ # download model RUN python -m pip install -U huggingface_hub[cli,hf_transfer] ARG REPO_ID="hkunlp/instructor-xl" ARG REVISION="ce48b213095e647a6c3536364b9fa00daf57f436" -COPY scripts/model_download.py scripts/model_download.py +COPY packages/text-embeddings/scripts/model_download.py scripts/model_download.py RUN REPO_ID=${REPO_ID} REVISION=${REVISION} python scripts/model_download.py # hardened and slim python image @@ -34,7 +40,7 @@ WORKDIR /leapfrogai COPY --from=builder /leapfrogai/.venv/ /leapfrogai/.venv/ COPY --from=builder /leapfrogai/.model/ /leapfrogai/.model/ -COPY main.py . +COPY packages/text-embeddings/main.py . EXPOSE 50051:50051 diff --git a/packages/vllm/Dockerfile b/packages/vllm/Dockerfile index ca6d7cd6e..0063db421 100755 --- a/packages/vllm/Dockerfile +++ b/packages/vllm/Dockerfile @@ -1,4 +1,7 @@ +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk + FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 as builder +ARG SDK_DEST=src/leapfrogai_sdk/build # Set the config file defaults ARG PYTHON_VERSION=3.11.6 @@ -40,6 +43,7 @@ RUN apt-get -y install git python3-venv make build-essential libssl-dev zlib1g-d RUN chown -R nonroot /home/leapfrogai/ USER nonroot +# # create virtual environment for light-weight portability and minimal libraries RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv ENV PYENV_ROOT="/home/leapfrogai/.pyenv" ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" @@ -49,11 +53,12 @@ RUN python3 -m venv .venv ENV PATH="/home/leapfrogai/.venv/bin:$PATH" -# # create virtual environment for light-weight portability and minimal libraries -# # NOTE: We are copying to this filename because installing 'optional extras' from a wheel requires the absolute path to the wheel file (instead of a wildcard whl) -COPY build/*.whl build/ -COPY build/lfai_vllm*.whl . -RUN pip install lfai_vllm*.whl --no-index --find-links=build/ +COPY --from=sdk /leapfrogai/${SDK_DEST} ./${SDK_DEST} +COPY packages/vllm packages/vllm + +RUN rm -f packages/vllm/build/*.whl +RUN python -m pip wheel packages/vllm -w packages/vllm/build --find-links=${SDK_DEST} +RUN pip install packages/vllm/build/lfai_vllm*.whl --no-index --find-links=packages/vllm/build/ # download model ENV HF_HOME=/home/leapfrogai/.cache/huggingface diff --git a/packages/whisper/Dockerfile b/packages/whisper/Dockerfile index 2555ed2d4..24d06d536 100644 --- a/packages/whisper/Dockerfile +++ b/packages/whisper/Dockerfile @@ -1,22 +1,29 @@ +ARG ARCH +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk + FROM --platform=$BUILDPLATFORM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev as builder +USER root +ARG SDK_DEST=src/leapfrogai_sdk/build WORKDIR /leapfrogai -COPY build/*.whl build/ - # create virtual environment for light-weight portability and minimal libraries RUN python3.11 -m venv .venv ENV PATH="/leapfrogai/.venv/bin:$PATH" +COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} +COPY packages/whisper packages/whisper + +RUN rm -f packages/whisper/build/*.whl +RUN python -m pip wheel "packages/whisper[dev]" -w packages/whisper/build --find-links=${SDK_DEST} + # download and covnert OpenAI's whisper base ARG MODEL_NAME=openai/whisper-base -RUN pip install ctranslate2 transformers[torch] --no-index --find-links=build/ +RUN pip install ctranslate2 transformers[torch] --no-index --find-links=packages/whisper/build/ RUN ct2-transformers-converter --model ${MODEL_NAME} --output_dir .model --copy_files tokenizer.json --quantization float32 RUN pip uninstall -y ctranslate2 transformers[torch] -COPY build/lfai_whisper*.whl . - -RUN pip install lfai_whisper*.whl --no-index --find-links=build/ +RUN pip install packages/whisper/build/lfai_whisper*.whl --no-index --find-links=packages/whisper/build/ # Use hardened ffmpeg image to get compiled binaries FROM cgr.dev/chainguard/ffmpeg:latest as ffmpeg @@ -40,7 +47,7 @@ ENV LD_LIBRARY_PATH \ /leapfrogai/.venv/lib64/python3.11/site-packages/nvidia/cublas/lib:\ /leapfrogai/.venv/lib64/python3.11/site-packages/nvidia/cudnn/lib -COPY main.py . +COPY packages/whisper/main.py . EXPOSE 50051:50051 diff --git a/src/leapfrogai_sdk/Dockerfile b/src/leapfrogai_sdk/Dockerfile new file mode 100644 index 000000000..4a9ec424a --- /dev/null +++ b/src/leapfrogai_sdk/Dockerfile @@ -0,0 +1,13 @@ +ARG ARCH=amd64 +FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder + +ARG SDK_DEST=src/leapfrogai_sdk/build +USER root +WORKDIR /leapfrogai + +COPY ./src/leapfrogai_sdk /leapfrogai/src/leapfrogai_sdk + +RUN python -m venv .venv +ENV PATH="/leapfrogai/.venv/bin:$PATH" +RUN rm -f ${SDK_DEST}/*.whl +RUN python -m pip wheel src/leapfrogai_sdk -w ${SDK_DEST} \ No newline at end of file From 7337cf28cc1874b96dcc3575def22646e94c2a0a Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Mon, 17 Jun 2024 15:00:08 -0700 Subject: [PATCH 02/19] Refactor so architecture is decided based on the -a flag and respected. Also modify the supabase chart values to not specify image versions in order to prevent versioning drift from the base chart (reduces number of places we need to change to just be the images in the zarf.yaml) --- Makefile | 3 +- README.md | 44 ++++++++++++++++++------- packages/api/zarf.yaml | 1 - packages/supabase/bitnami-values.yaml | 19 +---------- packages/supabase/zarf.yaml | 26 +++++++-------- packages/ui/zarf.yaml | 1 - src/leapfrogai_ui/.dockerignore | 47 ++++++++++++++++++++++++++- 7 files changed, 95 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 377185eb2..a756b5768 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ ARCH ?= amd64 +export ZARF_ARCHITECTURE = $(ARCH) KEY ?= "" REG_PORT ?= 5000 @@ -44,7 +45,7 @@ build-supabase: local-registry docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} ## Build the Zarf package - uds zarf package create packages/supabase -o packages/supabase --registry-override=ghcr.io=localhost:${REG_PORT} --set IMAGE_VERSION=${LOCAL_VERSION} --confirm + uds zarf package create packages/supabase -a ${ARCH} -o packages/supabase --registry-override=ghcr.io=localhost:${REG_PORT} --set IMAGE_VERSION=${LOCAL_VERSION} --confirm setup-api-deps: sdk-wheel ## Download the wheels for the leapfrogai_api dependencies diff --git a/README.md b/README.md index c5f8ff089..cbe11af7f 100644 --- a/README.md +++ b/README.md @@ -136,12 +136,14 @@ python -m venv .venv source .venv/bin/activate ``` +#### Linux and Windows (via WSL2) + Each component is built into its own Zarf package. You can build all of the packages you need at once with the following `Make` targets: ```shell -make build-cpu # api, llama-cpp-python, text-embeddings, whisper -make build-gpu # api, vllm, text-embeddings, whisper -make build-all # all of the backends +LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper +LOCAL_VERSION=dev make build-gpu # api, vllm, text-embeddings, whisper +LOCAL_VERSION=dev make build-all # all of the backends ``` **OR** @@ -149,22 +151,42 @@ make build-all # all of the backends You can build components individually using the following `Make` targets: ```shell -make build-api -make build-vllm # if you have GPUs -make build-llama-cpp-python # if you have CPU only -make build-text-embeddings -make build-whisper +LOCAL_VERSION=dev make build-api +LOCAL_VERSION=dev make build-vllm # if you have GPUs +LOCAL_VERSION=dev make build-llama-cpp-python # if you have CPU only +LOCAL_VERSION=dev make build-text-embeddings +LOCAL_VERSION=dev make build-whisper ``` +**NOTE: If you do not prepend your commands with `LOCAL_VERSION=dev`, uds will not find the generated zarf packages, as +they will be tagged with your current git hash instead of `dev` which uds expects** + +#### MacOS + +To run the same commands in MacOS, you will need to prepend your command with a couple of env vars like so: +All Macs: `REG_PORT=5001` +Apple Silicon (M1/M2/M3/M4 series) Macs: `ARCH=arm64` + +To demonstrate what this would look like for an Apple Silicon Mac: +``` shell +REG_PORT=5001 ARCH=arm64 LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper +``` + +To demonstrate what this would look like for an older Intel Mac: +``` shell +REG_PORT=5001 LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper +``` +Some containers do not support arm64 and therefore have their architecture specifiers hardcoded to amd64 (as of the of this writing, this is only Supabase: [issue](https://github.com/bitnami/charts/issues/22085)) + Once the packages are created, you can deploy either a CPU or GPU-enabled deployment via one of the UDS bundles: #### CPU ```shell cd uds-bundles/dev/cpu -uds create . -uds deploy k3d-core-slim-dev:0.22.2 -uds deploy uds-bundle-leapfrogai*.tar.zst +uds create -c . +uds deploy -c k3d-core-slim-dev:0.22.2 +uds deploy -c uds-bundle-leapfrogai*.tar.zst ``` #### GPU diff --git a/packages/api/zarf.yaml b/packages/api/zarf.yaml index 2c3755e63..1ceb87b5d 100644 --- a/packages/api/zarf.yaml +++ b/packages/api/zarf.yaml @@ -5,7 +5,6 @@ metadata: description: "LeapfrogAI" name: leapfrogai-api version: "###ZARF_PKG_TMPL_LEAPFROGAI_IMAGE_VERSION###" - architecture: amd64 constants: - name: LEAPFROGAI_API_VERSION diff --git a/packages/supabase/bitnami-values.yaml b/packages/supabase/bitnami-values.yaml index e905b6c22..ae8a61c16 100644 --- a/packages/supabase/bitnami-values.yaml +++ b/packages/supabase/bitnami-values.yaml @@ -21,10 +21,6 @@ commonLabels: jwt: autoGenerate: - image: - tag: 6.0.0-debian-12-r19 - kubectlImage: - tag: 1.29.3-debian-12-r4 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" @@ -59,8 +55,6 @@ auth: GOTRUE_EXTERNAL_KEYCLOAK_CLIENT_ID: "{{ .Values.leapfrogai.sso.clientId }}" GOTRUE_EXTERNAL_KEYCLOAK_REDIRECT_URI: "https://{{ .Values.leapfrogai.package.host }}.###ZARF_VAR_HOSTED_DOMAIN###/auth/v1/callback" GOTRUE_EXTERNAL_KEYCLOAK_URL: "https://sso.###ZARF_VAR_HOSTED_DOMAIN###/realms/uds" - image: - tag: 2.149.0-debian-12-r0 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" @@ -72,32 +66,24 @@ auth: key: secret meta: enabled: ###ZARF_VAR_ENABLE_META### - image: - tag: 0.80.0-debian-12-r1 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" realtime: enabled: ###ZARF_VAR_ENABLE_REALTIME### - image: - tag: 2.28.22-debian-12-r0 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" rest: enabled: ###ZARF_VAR_ENABLE_REST### - image: - tag: 11.2.2-debian-12-r14 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" storage: enabled: ###ZARF_VAR_ENABLE_STORAGE### - image: - tag: 0.48.4-debian-12-r0 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" @@ -105,8 +91,6 @@ storage: studio: enabled: ###ZARF_VAR_ENABLE_STUDIO### publicURL: "https://ai.###ZARF_VAR_HOSTED_DOMAIN###" - image: - tag: 0.24.3-debian-12-r0 resourcesPreset: "none" podLabels: sidecar.istio.io/inject: "false" @@ -118,7 +102,7 @@ volumePermissions: resourcesPreset: "none" psqlImage: - tag: 15.1.1-debian-12-r24 + tag: 15.1.1-debian-12-r34 kong: enabled: ###ZARF_VAR_ENABLE_KONG### @@ -181,7 +165,6 @@ kong: postgresql: enabled: ###ZARF_VAR_ENABLE_POSTGRES### image: - tag: 15.1.1-debian-12-r24 debug: true primary: resourcesPreset: "none" diff --git a/packages/supabase/zarf.yaml b/packages/supabase/zarf.yaml index 03801da63..74a1533b3 100644 --- a/packages/supabase/zarf.yaml +++ b/packages/supabase/zarf.yaml @@ -63,7 +63,7 @@ components: namespace: leapfrogai url: oci://registry-1.docker.io/bitnamicharts/supabase releaseName: supabase-bootstrap - version: 4.0.1 + version: 4.0.5 valuesFiles: - "bitnami-values.yaml" - "bitnami-values-bootstrap.yaml" @@ -78,21 +78,21 @@ components: namespace: leapfrogai url: oci://registry-1.docker.io/bitnamicharts/supabase releaseName: supabase - version: 4.0.1 + version: 4.0.5 valuesFiles: - "bitnami-values.yaml" images: - - docker.io/bitnami/gotrue:2.149.0-debian-12-r0 - - docker.io/bitnami/jwt-cli:6.0.0-debian-12-r19 - - docker.io/bitnami/kubectl:1.29.3-debian-12-r4 - - docker.io/bitnami/os-shell:12-debian-12-r18 - - docker.io/bitnami/postgrest:11.2.2-debian-12-r14 - - docker.io/bitnami/supabase-postgres:15.1.1-debian-12-r24 - - docker.io/bitnami/supabase-postgres-meta:0.80.0-debian-12-r1 - - docker.io/bitnami/supabase-realtime:2.28.22-debian-12-r0 - - docker.io/bitnami/supabase-storage:0.48.4-debian-12-r0 - - docker.io/bitnami/supabase-studio:0.24.3-debian-12-r0 - - docker.io/bitnami/kong:3.6.1-debian-12-r13 + - docker.io/bitnami/gotrue:2.150.1-debian-12-r1 + - docker.io/bitnami/jwt-cli:6.0.0-debian-12-r20 + - docker.io/bitnami/kubectl:1.30.0-debian-12-r0 + - docker.io/bitnami/os-shell:12-debian-12-r19 + - docker.io/bitnami/postgrest:11.2.2-debian-12-r15 + - docker.io/bitnami/supabase-postgres:15.1.1-debian-12-r34 + - docker.io/bitnami/supabase-postgres-meta:0.80.0-debian-12-r3 + - docker.io/bitnami/supabase-realtime:2.28.32-debian-12-r2 + - docker.io/bitnami/supabase-storage:0.48.4-debian-12-r2 + - docker.io/bitnami/supabase-studio:0.24.3-debian-12-r3 + - docker.io/bitnami/kong:3.6.1-debian-12-r18 - name: supabase-post-process description: "Perform necessary post processing here" required: true diff --git a/packages/ui/zarf.yaml b/packages/ui/zarf.yaml index 4a08a8c41..7f1ea4497 100644 --- a/packages/ui/zarf.yaml +++ b/packages/ui/zarf.yaml @@ -2,7 +2,6 @@ kind: ZarfPackageConfig metadata: name: leapfrogai-ui version: '###ZARF_PKG_TMPL_IMAGE_VERSION###' - architecture: amd64 description: > A UI for LeapfrogAI diff --git a/src/leapfrogai_ui/.dockerignore b/src/leapfrogai_ui/.dockerignore index 6e095c184..2b94c8767 100644 --- a/src/leapfrogai_ui/.dockerignore +++ b/src/leapfrogai_ui/.dockerignore @@ -13,6 +13,9 @@ Dockerfile .git .gitignore .gitattributes +.github/ +adr/ +tests/ README.md .npmrc .prettierrc @@ -20,6 +23,7 @@ README.md .editorconfig .vscode .idea +.venv build package @@ -31,4 +35,45 @@ zarf-sbom/ playwright/.auth -supabase/ \ No newline at end of file +supabase/ + +# Copied from .gitignore +data/* +*.tar.zst +__pycache__ +.venv +.ipynb_checkpoints +.vscode/ +venv +.DS_Store +dist/ +tmp/ +temp/ +zarf-sbom/ +/values.yaml +.pytest_cache +*egg-info +egg-info/ +build/ +*.whl +.model/ +*.gguf +.env +.ruff_cache +.branches +.temp +src/leapfrogai_api/config.yaml + +# local model and tokenizer files +*.bin +pytorch_model.bin.index.json +special_tokens_map.json +tokenizer_config.json +tokenizer.json +config.json +generation_config.json +vocabulary.json +.model/ + +# go binaries +api/main \ No newline at end of file From d710b2d629725e6748bd9ae4a595be301090d7f7 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:20:50 -0700 Subject: [PATCH 03/19] Fix issue where migrations would hang --- packages/supabase/bitnami-values.yaml | 5 ++--- packages/supabase/zarf.yaml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/supabase/bitnami-values.yaml b/packages/supabase/bitnami-values.yaml index ae8a61c16..371e08715 100644 --- a/packages/supabase/bitnami-values.yaml +++ b/packages/supabase/bitnami-values.yaml @@ -97,12 +97,10 @@ studio: volumePermissions: enabled: ###ZARF_VAR_ENABLE_VOLUME_PERMISSIONS### - image: - tag: 12-debian-12-r18 resourcesPreset: "none" psqlImage: - tag: 15.1.1-debian-12-r34 + tag: 15.1.1-debian-12-r69 kong: enabled: ###ZARF_VAR_ENABLE_KONG### @@ -165,6 +163,7 @@ kong: postgresql: enabled: ###ZARF_VAR_ENABLE_POSTGRES### image: + tag: 15.1.1-debian-12-r69 debug: true primary: resourcesPreset: "none" diff --git a/packages/supabase/zarf.yaml b/packages/supabase/zarf.yaml index 74a1533b3..fb01e2a09 100644 --- a/packages/supabase/zarf.yaml +++ b/packages/supabase/zarf.yaml @@ -87,7 +87,7 @@ components: - docker.io/bitnami/kubectl:1.30.0-debian-12-r0 - docker.io/bitnami/os-shell:12-debian-12-r19 - docker.io/bitnami/postgrest:11.2.2-debian-12-r15 - - docker.io/bitnami/supabase-postgres:15.1.1-debian-12-r34 + - docker.io/bitnami/supabase-postgres:15.1.1-debian-12-r69 - docker.io/bitnami/supabase-postgres-meta:0.80.0-debian-12-r3 - docker.io/bitnami/supabase-realtime:2.28.32-debian-12-r2 - docker.io/bitnami/supabase-storage:0.48.4-debian-12-r2 From b022dcff158ff382fe942229a188da1890e44bae Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:38:43 -0700 Subject: [PATCH 04/19] Fix build for make target build-repeater due to the sdk build refactor Remove dangling unnecesary make targets --- Makefile | 33 +++++++-------------------------- packages/api/Dockerfile | 1 - packages/repeater/Dockerfile | 20 ++++++++++++-------- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index a756b5768..e7e79ae3c 100644 --- a/Makefile +++ b/Makefile @@ -47,9 +47,7 @@ build-supabase: local-registry ## Build the Zarf package uds zarf package create packages/supabase -a ${ARCH} -o packages/supabase --registry-override=ghcr.io=localhost:${REG_PORT} --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -setup-api-deps: sdk-wheel ## Download the wheels for the leapfrogai_api dependencies - -build-api: local-registry setup-api-deps ## Build the leapfrogai_api container and Zarf package +build-api: local-registry sdk-wheel ## Build the leapfrogai_api container and Zarf package ## Build the API image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} @@ -82,10 +80,7 @@ build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package ## Build the Zarf package uds zarf package create packages/ui -o packages/ui --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm - -setup-llama-cpp-python-deps: sdk-wheel ## Download the wheels for the optional 'llama-cpp-python' dependencies - -build-llama-cpp-python: local-registry setup-llama-cpp-python-deps ## Build the llama-cpp-python (cpu) container and Zarf package +build-llama-cpp-python: local-registry sdk-wheel ## Build the llama-cpp-python (cpu) container and Zarf package ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} -f packages/llama-cpp-python/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} @@ -96,11 +91,7 @@ build-llama-cpp-python: local-registry setup-llama-cpp-python-deps ## Build the ## Build the Zarf package uds zarf package create packages/llama-cpp-python -o packages/llama-cpp-python --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm - -setup-vllm-deps: sdk-wheel ## Download the wheels for the optional 'vllm' dependencies - - -build-vllm: local-registry setup-vllm-deps ## Build the vllm container and Zarf package +build-vllm: local-registry sdk-wheel ## Build the vllm container and Zarf package ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} -f packages/vllm/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} @@ -111,11 +102,7 @@ build-vllm: local-registry setup-vllm-deps ## Build the vllm container and Zarf ## Build the Zarf package uds zarf package create packages/vllm -o packages/vllm --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm - -setup-text-embeddings-deps: sdk-wheel ## Download the wheels for the optional 'text-embeddings' dependencies - - -build-text-embeddings: local-registry setup-text-embeddings-deps ## Build the text-embeddings container and Zarf package +build-text-embeddings: local-registry sdk-wheel ## Build the text-embeddings container and Zarf package ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} -f packages/text-embeddings/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} @@ -127,9 +114,7 @@ build-text-embeddings: local-registry setup-text-embeddings-deps ## Build the te uds zarf package create packages/text-embeddings -o packages/text-embeddings --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -setup-whisper-deps: sdk-wheel ## Download the wheels for the optional 'whisper' dependencies - -build-whisper: local-registry setup-whisper-deps ## Build the whisper container and zarf package +build-whisper: local-registry sdk-wheel ## Build the whisper container and zarf package ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} -f packages/whisper/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} @@ -140,13 +125,9 @@ build-whisper: local-registry setup-whisper-deps ## Build the whisper container ## Build the Zarf package uds zarf package create packages/whisper -o packages/whisper --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -setup-repeater-deps: sdk-wheel ## Download the wheels for the optional 'repeater' dependencies - -rm packages/repeater/build/*.whl - python -m pip wheel packages/repeater -w packages/repeater/build --find-links=${SDK_DEST} - -build-repeater: local-registry setup-repeater-deps ## Build the repeater container and zarf package +build-repeater: local-registry sdk-wheel ## Build the repeater container and zarf package ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} packages/repeater + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} -f packages/repeater/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 99559cc48..43cc7fc2b 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -7,7 +7,6 @@ USER root WORKDIR /leapfrogai # copy the api dependencies over -# NOTE: We are copying to this filename because installing 'optional extras' from a wheel requires the absolute path to the wheel file (instead of a wildcard whl) COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} COPY src/leapfrogai_api src/leapfrogai_api diff --git a/packages/repeater/Dockerfile b/packages/repeater/Dockerfile index bbc98679e..67c50e421 100644 --- a/packages/repeater/Dockerfile +++ b/packages/repeater/Dockerfile @@ -1,18 +1,22 @@ -ARG ARCH=amd64 +ARG ARCH +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk # hardened and slim python w/ developer tools image FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder - +ARG SDK_DEST=src/leapfrogai_sdk/build +USER root WORKDIR /leapfrogai +COPY --from=sdk /leapfrogai/${SDK_DEST} ${SDK_DEST} +COPY packages/repeater packages/repeater + # create virtual environment for light-weight portability and minimal libraries -RUN python3.11 -m venv .venv +RUN python -m venv .venv ENV PATH="/leapfrogai/.venv/bin:$PATH" -# copy and install all python dependencies -COPY build/*.whl build/ -COPY build/lfai_repeater*.whl . -RUN pip install lfai_repeater*.whl --no-index --find-links=build/ +RUN rm -f packages/repeater/build/*.whl +RUN python -m pip wheel packages/repeater -w packages/repeater/build --find-links=${SDK_DEST} +RUN pip install packages/repeater/build/lfai_repeater*.whl --no-index --find-links=packages/repeater/build/ # hardened and slim python image FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-${ARCH} @@ -23,7 +27,7 @@ WORKDIR /leapfrogai COPY --from=builder /leapfrogai/.venv/ /leapfrogai/.venv/ -COPY repeater.py . +COPY packages/repeater/repeater.py . # Publish port EXPOSE 50051:50051 From 9ce85863d76f5138d8c445ec11e7e99dadf35ee4 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:54:17 -0700 Subject: [PATCH 05/19] Refactor Makefile to abstract the docker build from the zarf package to simplify the ci build without a divergent pattern, as well as allow quicker dev iteration on docker builds --- .github/workflows/pytest.yaml | 8 +++----- Makefile | 26 +++++++++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 8abf7ed09..abbba201d 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -48,12 +48,10 @@ jobs: python-version-file: 'pyproject.toml' - name: Build Repeater + env: + LOCAL_VERSION: dev run: | - make local-registry - make setup-repeater-deps - docker build -t ghcr.io/defenseunicorns/leapfrogai/repeater:dev packages/repeater - docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:dev localhost:5000/defenseunicorns/leapfrogai/repeater:dev - + make build-repeater - name: Run Repeater run: docker run -p 50051:50051 -d --name=repeater ghcr.io/defenseunicorns/leapfrogai/repeater:dev diff --git a/Makefile b/Makefile index e7e79ae3c..82b98ab3d 100644 --- a/Makefile +++ b/Makefile @@ -38,16 +38,18 @@ local-registry: ## Start up a local container registry. Errors in this target ar sdk-wheel: ## build wheels for the leapfrogai_sdk package as a dependency for other lfai components docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest -f src/leapfrogai_sdk/Dockerfile . -build-supabase: local-registry +docker-supabase: local-registry ## Build the migration container for this version of the supabase package docker build -t ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/supabase/migrations" . docker tag ghcr.io/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} + +build-supabase: docker-supabase docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/supabase-migrations:${LOCAL_VERSION} ## Build the Zarf package uds zarf package create packages/supabase -a ${ARCH} -o packages/supabase --registry-override=ghcr.io=localhost:${REG_PORT} --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-api: local-registry sdk-wheel ## Build the leapfrogai_api container and Zarf package +docker-api: local-registry sdk-wheel ## Build the API image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} @@ -56,6 +58,7 @@ build-api: local-registry sdk-wheel ## Build the leapfrogai_api container and Za docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . docker tag ghcr.io/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} +build-api: docker-api ## Build the leapfrogai_api container and Zarf package ## Push the images to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/api-migrations:${LOCAL_VERSION} @@ -63,8 +66,7 @@ build-api: local-registry sdk-wheel ## Build the leapfrogai_api container and Za ## Build the Zarf package uds zarf package create packages/api -o packages/api --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set LEAPFROGAI_IMAGE_VERSION=${LOCAL_VERSION} --confirm - -build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package +docker-ui: local-registry ## Build the UI image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} src/leapfrogai_ui docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} @@ -73,6 +75,7 @@ build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" . docker tag ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} +build-ui: docker-ui ## Build the leapfrogai_ui container and Zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-ui:${LOCAL_VERSION} docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/ui-migrations:${LOCAL_VERSION} @@ -80,33 +83,36 @@ build-ui: local-registry ## Build the leapfrogai_ui container and Zarf package ## Build the Zarf package uds zarf package create packages/ui -o packages/ui --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-llama-cpp-python: local-registry sdk-wheel ## Build the llama-cpp-python (cpu) container and Zarf package +docker-llama-cpp-python: local-registry sdk-wheel ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} -f packages/llama-cpp-python/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} +build-llama-cpp-python: docker-llama-cpp-python ## Build the llama-cpp-python (cpu) container and Zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} ## Build the Zarf package uds zarf package create packages/llama-cpp-python -o packages/llama-cpp-python --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-vllm: local-registry sdk-wheel ## Build the vllm container and Zarf package +docker-vllm: local-registry sdk-wheel ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} -f packages/vllm/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} +build-vllm: docker-vllm ## Build the vllm container and Zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} ## Build the Zarf package uds zarf package create packages/vllm -o packages/vllm --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-text-embeddings: local-registry sdk-wheel ## Build the text-embeddings container and Zarf package +docker-text-embeddings: local-registry sdk-wheel ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} -f packages/text-embeddings/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} +build-text-embeddings: docker-text-embeddings ## Build the text-embeddings container and Zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} @@ -114,22 +120,24 @@ build-text-embeddings: local-registry sdk-wheel ## Build the text-embeddings con uds zarf package create packages/text-embeddings -o packages/text-embeddings --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-whisper: local-registry sdk-wheel ## Build the whisper container and zarf package +docker-whisper: local-registry sdk-wheel ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} -f packages/whisper/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} +build-whisper: docker-whisper ## Build the whisper container and zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} ## Build the Zarf package uds zarf package create packages/whisper -o packages/whisper --registry-override=ghcr.io=localhost:${REG_PORT} --insecure --set IMAGE_VERSION=${LOCAL_VERSION} --confirm -build-repeater: local-registry sdk-wheel ## Build the repeater container and zarf package +docker-repeater: local-registry sdk-wheel ## Build the image (and tag it for the local registry) docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} -f packages/repeater/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} +build-repeater: docker-repeater ## Build the repeater container and zarf package ## Push the image to the local registry (Zarf is super slow if the image is only in the local daemon) docker push localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} From 2cd30d728b9b1aaba1be253099996f62437989a3 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:57:02 -0700 Subject: [PATCH 06/19] Fix wrong make target after the refactor --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index abbba201d..1dda8ce1a 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -51,7 +51,7 @@ jobs: env: LOCAL_VERSION: dev run: | - make build-repeater + make docker-repeater - name: Run Repeater run: docker run -p 50051:50051 -d --name=repeater ghcr.io/defenseunicorns/leapfrogai/repeater:dev From 8cb5e0e743e018204baf3843e1c0245b2a69d008 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:06:02 -0700 Subject: [PATCH 07/19] Fix release.yaml that would break after the refactoring of the docker build --- .github/workflows/release.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 85ee57daa..996cceeb8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,14 +41,14 @@ jobs: python-version-file: 'pyproject.toml' - name: Download Python Wheels - run: make setup-api-deps setup-repeater-deps setup-llama-cpp-python-deps setup-vllm-deps setup-text-embeddings-deps setup-whisper-deps + run: make sdk-wheel - name: Install Zarf uses: defenseunicorns/setup-zarf@f95763914e20e493bb5d45d63e30e17138f981d6 # v1.0.0 - name: Build and Publish API run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push packages/api + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push -f packages/api/Dockerfile . docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm @@ -62,7 +62,7 @@ jobs: - name: Build and Publish UI run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${{ steps.get_version.outputs.version-without-v }} --push src/leapfrogai_ui + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-ui:${{ steps.get_version.outputs.version-without-v }} --push -f src/leapfrogai_ui/Dockerfile . docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/ui-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=src/leapfrogai_ui/supabase/migrations" . zarf package create packages/ui --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm @@ -86,7 +86,7 @@ jobs: - name: Build and Publish repeater run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push packages/repeater + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push -f packages/repeater/Dockerfile . zarf package create packages/repeater --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/repeater --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -99,7 +99,7 @@ jobs: - name: Build and Publish llama run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push packages/llama-cpp-python + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push -f packages/llama-cpp-python/Dockerfile . zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -112,7 +112,7 @@ jobs: - name: Build and Publish vLLM run: | - docker buildx build -t ghcr.io/defenseunicorns/leapfrogai/vllm:${{ steps.get_version.outputs.version-without-v }} --push packages/vllm + docker buildx build -t ghcr.io/defenseunicorns/leapfrogai/vllm:${{ steps.get_version.outputs.version-without-v }} --push -f packages/vllm/Dockerfile . zarf package create packages/vllm --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --confirm @@ -123,7 +123,7 @@ jobs: - name: Build and Publish Text-Embeddings run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ steps.get_version.outputs.version-without-v }} --push packages/text-embeddings + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ steps.get_version.outputs.version-without-v }} --push -f packages/text-embeddings/Dockerfile . zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -136,7 +136,7 @@ jobs: - name: Build and Publish whisper run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ steps.get_version.outputs.version-without-v }} --push packages/whisper + docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ steps.get_version.outputs.version-without-v }} --push -f packages/whisper/Dockerfile . zarf package create packages/whisper --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/whisper --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm From 16e4738695d42941278ce172497a9f7b85102876 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:19:42 -0700 Subject: [PATCH 08/19] Change MacOS to the official formatting `macOS` Add some formatting to README.md to make it hopefully more readable and clear Remove duplicate .dockerignore entries --- README.md | 16 +++++++++++----- src/leapfrogai_ui/.dockerignore | 7 ------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index cbe11af7f..48458ec4e 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ You can build components individually using the following `Make` targets: ```shell LOCAL_VERSION=dev make build-api -LOCAL_VERSION=dev make build-vllm # if you have GPUs +LOCAL_VERSION=dev make build-vllm # if you have GPUs (macOS not supported) LOCAL_VERSION=dev make build-llama-cpp-python # if you have CPU only LOCAL_VERSION=dev make build-text-embeddings LOCAL_VERSION=dev make build-whisper @@ -161,10 +161,12 @@ LOCAL_VERSION=dev make build-whisper **NOTE: If you do not prepend your commands with `LOCAL_VERSION=dev`, uds will not find the generated zarf packages, as they will be tagged with your current git hash instead of `dev` which uds expects** -#### MacOS +#### macOS + +To run the same commands in macOS, you will need to prepend your command with a couple of env vars like so: -To run the same commands in MacOS, you will need to prepend your command with a couple of env vars like so: All Macs: `REG_PORT=5001` + Apple Silicon (M1/M2/M3/M4 series) Macs: `ARCH=arm64` To demonstrate what this would look like for an Apple Silicon Mac: @@ -176,9 +178,13 @@ To demonstrate what this would look like for an older Intel Mac: ``` shell REG_PORT=5001 LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper ``` -Some containers do not support arm64 and therefore have their architecture specifiers hardcoded to amd64 (as of the of this writing, this is only Supabase: [issue](https://github.com/bitnami/charts/issues/22085)) -Once the packages are created, you can deploy either a CPU or GPU-enabled deployment via one of the UDS bundles: +**OR** + +You can build components individually using the following `Make` targets, just like in the Linux section except ensuring +to prepend the env vars detailed above. + +### Once the packages are created, you can deploy either a CPU or GPU-enabled deployment via one of the UDS bundles (macOS only supports cpu): #### CPU diff --git a/src/leapfrogai_ui/.dockerignore b/src/leapfrogai_ui/.dockerignore index 2b94c8767..b2836e9fb 100644 --- a/src/leapfrogai_ui/.dockerignore +++ b/src/leapfrogai_ui/.dockerignore @@ -1,7 +1,6 @@ .DS_Store node_modules /.svelte-kit -/build /package **/.env !.env.public @@ -21,20 +20,14 @@ README.md .prettierrc .eslintrc.cjs .editorconfig -.vscode .idea -.venv -build package - *.tar.zst zarf-sbom/ - playwright/.auth - supabase/ # Copied from .gitignore From c4eb4e9a90d11efa7e111622b7a6fe45b789477c Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:26:24 -0700 Subject: [PATCH 09/19] remove duplicate .dockerignore entries if they're included in the .gitignore copied section --- src/leapfrogai_ui/.dockerignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/leapfrogai_ui/.dockerignore b/src/leapfrogai_ui/.dockerignore index b2836e9fb..357be5ec8 100644 --- a/src/leapfrogai_ui/.dockerignore +++ b/src/leapfrogai_ui/.dockerignore @@ -1,4 +1,3 @@ -.DS_Store node_modules /.svelte-kit /package @@ -21,16 +20,15 @@ README.md .eslintrc.cjs .editorconfig .idea -package *.tar.zst -zarf-sbom/ playwright/.auth supabase/ # Copied from .gitignore +# temporary deployment and development artifacts data/* *.tar.zst __pycache__ @@ -56,6 +54,8 @@ build/ .branches .temp src/leapfrogai_api/config.yaml +.idea/ +/.python-version # local model and tokenizer files *.bin @@ -69,4 +69,4 @@ vocabulary.json .model/ # go binaries -api/main \ No newline at end of file +api/main From 94253f4e7edacbafb1fc91623ab32b76a0c63bd3 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 18 Jun 2024 18:37:21 -0700 Subject: [PATCH 10/19] Fix vllm build after refactor --- packages/vllm/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/vllm/Dockerfile b/packages/vllm/Dockerfile index 0063db421..d631473d0 100755 --- a/packages/vllm/Dockerfile +++ b/packages/vllm/Dockerfile @@ -40,7 +40,10 @@ RUN apt-get -y update \ # get deps for vllm compilation, model download, and pyenv RUN apt-get -y install git python3-venv make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev tk-dev libffi-dev +COPY --from=sdk /leapfrogai/${SDK_DEST} ./${SDK_DEST} +COPY packages/vllm packages/vllm RUN chown -R nonroot /home/leapfrogai/ + USER nonroot # # create virtual environment for light-weight portability and minimal libraries @@ -52,17 +55,13 @@ RUN pyenv global ${PYTHON_VERSION} RUN python3 -m venv .venv ENV PATH="/home/leapfrogai/.venv/bin:$PATH" - -COPY --from=sdk /leapfrogai/${SDK_DEST} ./${SDK_DEST} -COPY packages/vllm packages/vllm - RUN rm -f packages/vllm/build/*.whl RUN python -m pip wheel packages/vllm -w packages/vllm/build --find-links=${SDK_DEST} RUN pip install packages/vllm/build/lfai_vllm*.whl --no-index --find-links=packages/vllm/build/ # download model ENV HF_HOME=/home/leapfrogai/.cache/huggingface -COPY src/ src/ +COPY packages/vllm/src/ src/ # Load ARG values into env variables for pickup by confz ENV LAI_HF_HUB_ENABLE_HF_TRANSFER=${HF_HUB_ENABLE_HF_TRANSFER} From b692e006e5e21bef51d4d242e23195b2fc4dbfc0 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Thu, 20 Jun 2024 09:10:12 -0700 Subject: [PATCH 11/19] Address comments to changes in README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 48458ec4e..645380de7 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,9 @@ source .venv/bin/activate Each component is built into its own Zarf package. You can build all of the packages you need at once with the following `Make` targets: ```shell -LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper -LOCAL_VERSION=dev make build-gpu # api, vllm, text-embeddings, whisper -LOCAL_VERSION=dev make build-all # all of the backends +LOCAL_VERSION=dev make build-cpu +LOCAL_VERSION=dev make build-gpu +LOCAL_VERSION=dev make build-all ``` **OR** @@ -174,9 +174,9 @@ To demonstrate what this would look like for an Apple Silicon Mac: REG_PORT=5001 ARCH=arm64 LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper ``` -To demonstrate what this would look like for an older Intel Mac: +To demonstrate what this would look like for an older Intel Mac (not officially supported): ``` shell -REG_PORT=5001 LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper +REG_PORT=5001 LOCAL_VERSION=dev make build-cpu ``` **OR** From 1869014181341ba377352f530281a718c12a4e30 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Thu, 20 Jun 2024 19:31:25 -0700 Subject: [PATCH 12/19] refactor vllm Dockerfile to be multistage and use a smaller base image --- .dockerignore | 6 ++++ .gitignore | 4 +-- packages/vllm/Dockerfile | 62 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..37aa07168 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +# Copied from .gitignore +# temporary deployment and development artifacts +**/*.tar.zst +**/Dockerfile* +**/.gitignore +**/Makefile \ No newline at end of file diff --git a/.gitignore b/.gitignore index b7c0db10f..d2179c5e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # temporary deployment and development artifacts data/* -*.tar.zst +**/*.tar.zst __pycache__ .venv .ipynb_checkpoints @@ -16,7 +16,7 @@ zarf-sbom/ *egg-info egg-info/ build/ -*.whl +**/*.whl .model/ *.gguf .env diff --git a/packages/vllm/Dockerfile b/packages/vllm/Dockerfile index d631473d0..cbb602ce2 100755 --- a/packages/vllm/Dockerfile +++ b/packages/vllm/Dockerfile @@ -59,9 +59,65 @@ RUN rm -f packages/vllm/build/*.whl RUN python -m pip wheel packages/vllm -w packages/vllm/build --find-links=${SDK_DEST} RUN pip install packages/vllm/build/lfai_vllm*.whl --no-index --find-links=packages/vllm/build/ +FROM nvidia/cuda:12.2.2-runtime-ubuntu22.04 +## COPIED FROM ABOVE ## +ARG SDK_DEST=src/leapfrogai_sdk/build +# Set the config file defaults +ARG PYTHON_VERSION=3.11.6 +ARG HF_HUB_ENABLE_HF_TRANSFER="1" +ARG REPO_ID="TheBloke/Synthia-7B-v2.0-GPTQ" +ARG REVISION="gptq-4bit-32g-actorder_True" +ARG QUANTIZATION="gptq" +ARG MODEL_SOURCE=".model/" +ARG MAX_CONTEXT_LENGTH=32768 +ARG STOP_TOKENS='["","<|endoftext|>","<|im_end|>"]' +ARG PROMPT_FORMAT_CHAT_SYSTEM="SYSTEM: {}\n" +ARG PROMPT_FORMAT_CHAT_ASSISTANT="ASSISTANT: {}\n" +ARG PROMPT_FORMAT_CHAT_USER="USER: {}\n" +ARG PROMPT_FORMAT_DEFAULTS_TOP_P=1.0 +ARG PROMPT_FORMAT_DEFAULTS_TOP_K=0 +ARG TENSOR_PARALLEL_SIZE=1 + +ENV DEBIAN_FRONTEND=noninteractive + +USER root + +RUN groupadd -g 65532 vglusers && \ + useradd -ms /bin/bash nonroot -u 65532 -g 65532 && \ + usermod -a -G video,sudo nonroot + +WORKDIR /home/leapfrogai + +# grab necesary python dependencies +RUN apt-get -y update \ + && apt-get install -y software-properties-common \ + && add-apt-repository universe \ + && add-apt-repository ppa:deadsnakes/ppa \ + && apt-get -y update + +# get deps for vllm compilation, model download, and pyenv +RUN apt-get -y install git python3-venv make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev tk-dev libffi-dev + +COPY --from=sdk /leapfrogai/${SDK_DEST} ./${SDK_DEST} +COPY packages/vllm packages/vllm +RUN chown -R nonroot /home/leapfrogai/ + +USER nonroot + +# # create virtual environment for light-weight portability and minimal libraries +RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv +ENV PYENV_ROOT="/home/leapfrogai/.pyenv" +ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" +RUN pyenv install ${PYTHON_VERSION} +RUN python3 -m venv .venv +ENV PATH="/home/leapfrogai/.venv/bin:$PATH" +## COPIED FROM ABOVE ## + +COPY --from=builder /home/leapfrogai/.venv /home/leapfrogai/.venv +COPY --from=builder /home/leapfrogai/packages/vllm/src /home/leapfrogai/packages/vllm/src + # download model ENV HF_HOME=/home/leapfrogai/.cache/huggingface -COPY packages/vllm/src/ src/ # Load ARG values into env variables for pickup by confz ENV LAI_HF_HUB_ENABLE_HF_TRANSFER=${HF_HUB_ENABLE_HF_TRANSFER} @@ -79,8 +135,8 @@ ENV LAI_PROMPT_FORMAT_DEFAULTS_TOP_P=${PROMPT_FORMAT_DEFAULTS_TOP_P} ENV LAI_PROMPT_FORMAT_DEFAULTS_TOP_K=${PROMPT_FORMAT_DEFAULTS_TOP_K} RUN pip install -U huggingface_hub[cli,hf_transfer] -RUN python3 src/model_download.py +RUN python3 packages/vllm/src/model_download.py EXPOSE 50051:50051 -ENTRYPOINT ["python", "-m", "leapfrogai_sdk.cli", "--app-dir=src/", "main:Model"] \ No newline at end of file +ENTRYPOINT ["python", "-m", "leapfrogai_sdk.cli", "--app-dir=packages/vllm/src/", "main:Model"] \ No newline at end of file From 26d86e17264b3631d39a7c30f54bef6dc649310e Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:23:25 -0700 Subject: [PATCH 13/19] remove old comment --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 37aa07168..729e30e3f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,3 @@ -# Copied from .gitignore -# temporary deployment and development artifacts **/*.tar.zst **/Dockerfile* **/.gitignore From e21229cdb2fc1c058343e4bbee034ad2096922ad Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Fri, 21 Jun 2024 09:25:10 -0700 Subject: [PATCH 14/19] update leapfrog_ui .dockerignore --- src/leapfrogai_ui/.dockerignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/leapfrogai_ui/.dockerignore b/src/leapfrogai_ui/.dockerignore index 357be5ec8..25e70ce40 100644 --- a/src/leapfrogai_ui/.dockerignore +++ b/src/leapfrogai_ui/.dockerignore @@ -30,7 +30,7 @@ supabase/ # Copied from .gitignore # temporary deployment and development artifacts data/* -*.tar.zst +**/*.tar.zst __pycache__ .venv .ipynb_checkpoints @@ -46,7 +46,7 @@ zarf-sbom/ *egg-info egg-info/ build/ -*.whl +**/*.whl .model/ *.gguf .env From 0f5c60b87169c72850824caddacbcecd217d894e Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:05:42 -0700 Subject: [PATCH 15/19] Remove unnecessary apt installs in final vllm docker image --- packages/vllm/Dockerfile | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/packages/vllm/Dockerfile b/packages/vllm/Dockerfile index cbb602ce2..52419b7ff 100755 --- a/packages/vllm/Dockerfile +++ b/packages/vllm/Dockerfile @@ -88,20 +88,14 @@ RUN groupadd -g 65532 vglusers && \ WORKDIR /home/leapfrogai -# grab necesary python dependencies -RUN apt-get -y update \ - && apt-get install -y software-properties-common \ - && add-apt-repository universe \ - && add-apt-repository ppa:deadsnakes/ppa \ - && apt-get -y update - -# get deps for vllm compilation, model download, and pyenv -RUN apt-get -y install git python3-venv make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev tk-dev libffi-dev - COPY --from=sdk /leapfrogai/${SDK_DEST} ./${SDK_DEST} -COPY packages/vllm packages/vllm +COPY --from=builder /home/leapfrogai/.venv /home/leapfrogai/.venv +COPY --from=builder /home/leapfrogai/packages/vllm/src /home/leapfrogai/packages/vllm/src RUN chown -R nonroot /home/leapfrogai/ +RUN apt-get -y update +RUN apt-get -y install git wget build-essential libssl-dev zlib1g-dev libffi-dev + USER nonroot # # create virtual environment for light-weight portability and minimal libraries @@ -109,12 +103,7 @@ RUN git clone --depth=1 https://github.com/pyenv/pyenv.git .pyenv ENV PYENV_ROOT="/home/leapfrogai/.pyenv" ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH" RUN pyenv install ${PYTHON_VERSION} -RUN python3 -m venv .venv ENV PATH="/home/leapfrogai/.venv/bin:$PATH" -## COPIED FROM ABOVE ## - -COPY --from=builder /home/leapfrogai/.venv /home/leapfrogai/.venv -COPY --from=builder /home/leapfrogai/packages/vllm/src /home/leapfrogai/packages/vllm/src # download model ENV HF_HOME=/home/leapfrogai/.cache/huggingface From 01450332a24694184b860d58e15cb62f1c85fead Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:11:50 -0700 Subject: [PATCH 16/19] migrate macos instructions to uds dev README.md --- uds-bundles/dev/README.md | 58 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/uds-bundles/dev/README.md b/uds-bundles/dev/README.md index f894ca60d..06968b629 100644 --- a/uds-bundles/dev/README.md +++ b/uds-bundles/dev/README.md @@ -11,12 +11,14 @@ python -m venv .venv source .venv/bin/activate ``` +#### Linux and Windows (via WSL2) + Each component is built into its own Zarf package. You can build all of the packages you need at once with the following `Make` targets: ```shell -make build-cpu # api, llama-cpp-python, text-embeddings, whisper, supabase -make build-gpu # api, vllm, text-embeddings, whisper, supabase -make build-all # all of the backends +LOCAL_VERSION=dev make build-cpu # api, llama-cpp-python, text-embeddings, whisper, supabase +LOCAL_VERSION=dev make build-gpu # api, vllm, text-embeddings, whisper, supabase +LOCAL_VERSION=dev make build-all # all of the backends ``` **OR** @@ -24,17 +26,45 @@ make build-all # all of the backends You can build components individually using the following `Make` targets: ```shell -make build-api -make build-supabase -make build-vllm # if you have GPUs -make build-llama-cpp-python # if you have CPU only -make build-text-embeddings -make build-whisper +LOCAL_VERSION=dev make build-api +LOCAL_VERSION=dev make build-supabase +LOCAL_VERSION=dev make build-vllm # if you have GPUs (macOS not supported) +LOCAL_VERSION=dev make build-llama-cpp-python # if you have CPU only +LOCAL_VERSION=dev make build-text-embeddings +LOCAL_VERSION=dev make build-whisper +``` + +**NOTE: If you do not prepend your commands with `LOCAL_VERSION=dev`, uds will not find the generated zarf packages, as +they will be tagged with your current git hash instead of `dev` which uds expects** + +#### macOS + +To run the same commands in macOS, you will need to prepend your command with a couple of env vars like so: + +All Macs: `REG_PORT=5001` + +Apple Silicon (M1/M2/M3/M4 series) Macs: `ARCH=arm64` + +To demonstrate what this would look like for an Apple Silicon Mac: +``` shell +REG_PORT=5001 ARCH=arm64 LOCAL_VERSION=dev make build-cpu ``` -Once the packages are created, you can deploy either a CPU or GPU-enabled deployment via one of the UDS bundles: +To demonstrate what this would look like for an older Intel Mac (not officially supported): +``` shell +REG_PORT=5001 LOCAL_VERSION=dev make build-cpu +``` + +**OR** + +You can build components individually using the following `Make` targets, just like in the Linux section except ensuring +to prepend the env vars detailed above. -## CPU +#### Once the packages are created, you can deploy either a CPU or GPU-enabled deployment via one of the UDS bundles (macOS only supports cpu) + +# Deploying via UDS bundle + +## CPU UDS Deployment Create the uds CPU bundle: ```shell @@ -42,14 +72,12 @@ cd uds-bundles/dev/cpu uds create . ``` -Deploy a [UDS cluster](/README.md#uds) if one isn't deployed already - Deploy the LeapfrogAI bundle: ```shell uds deploy uds-bundle-leapfrogai*.tar.zst ``` -## GPU +## GPU UDS Deployment Create the uds GPU bundle: ```shell @@ -57,8 +85,6 @@ cd uds-bundles/dev/gpu uds create . ``` -Deploy a [UDS cluster](/README.md#uds) with the following flags, as so: - ```shell uds deploy {k3d-cluster-name} --set K3D_EXTRA_ARGS="--gpus=all --image=ghcr.io/justinthelaw/k3d-gpu-support:v1.27.4-k3s1-cuda" ``` From c918d9f3913fd4e81d48064d110e61ebbef9c8d1 Mon Sep 17 00:00:00 2001 From: Mark Van Aken <1727697+vanakema@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:49:52 -0700 Subject: [PATCH 17/19] Address comments Fix issue with README.md changes --- Makefile | 16 +++++++--------- packages/api/Dockerfile | 3 ++- packages/llama-cpp-python/Dockerfile | 3 ++- packages/repeater/Dockerfile | 3 ++- packages/text-embeddings/Dockerfile | 3 ++- packages/vllm/Dockerfile | 3 ++- packages/whisper/Dockerfile | 3 ++- uds-bundles/dev/README.md | 5 +++++ 8 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 82b98ab3d..18e755855 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,9 @@ ARCH ?= amd64 -export ZARF_ARCHITECTURE = $(ARCH) KEY ?= "" REG_PORT ?= 5000 VERSION ?= $(shell git describe --abbrev=0 --tags) LOCAL_VERSION ?= $(shell git rev-parse --short HEAD) -SDK_DEST ?= src/leapfrogai_sdk/build ###################################################################################### .PHONY: help @@ -36,7 +34,7 @@ local-registry: ## Start up a local container registry. Errors in this target ar -docker run -d -p ${REG_PORT}:5000 --restart=always --name registry registry:2 sdk-wheel: ## build wheels for the leapfrogai_sdk package as a dependency for other lfai components - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest -f src/leapfrogai_sdk/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} -f src/leapfrogai_sdk/Dockerfile . docker-supabase: local-registry ## Build the migration container for this version of the supabase package @@ -51,7 +49,7 @@ build-supabase: docker-supabase docker-api: local-registry sdk-wheel ## Build the API image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg SDK_DEST=${SDK_DEST} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} -f packages/api/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/leapfrogai-api:${LOCAL_VERSION} ## Build the migration container for this version of the API @@ -85,7 +83,7 @@ build-ui: docker-ui ## Build the leapfrogai_ui container and Zarf package docker-llama-cpp-python: local-registry sdk-wheel ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} -f packages/llama-cpp-python/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} -f packages/llama-cpp-python/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/llama-cpp-python:${LOCAL_VERSION} build-llama-cpp-python: docker-llama-cpp-python ## Build the llama-cpp-python (cpu) container and Zarf package @@ -97,7 +95,7 @@ build-llama-cpp-python: docker-llama-cpp-python ## Build the llama-cpp-python (c docker-vllm: local-registry sdk-wheel ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} -f packages/vllm/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} -f packages/vllm/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/vllm:${LOCAL_VERSION} build-vllm: docker-vllm ## Build the vllm container and Zarf package @@ -109,7 +107,7 @@ build-vllm: docker-vllm ## Build the vllm container and Zarf package docker-text-embeddings: local-registry sdk-wheel ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} -f packages/text-embeddings/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} -f packages/text-embeddings/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/text-embeddings:${LOCAL_VERSION} build-text-embeddings: docker-text-embeddings ## Build the text-embeddings container and Zarf package @@ -122,7 +120,7 @@ build-text-embeddings: docker-text-embeddings ## Build the text-embeddings conta docker-whisper: local-registry sdk-wheel ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} -f packages/whisper/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} -f packages/whisper/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/whisper:${LOCAL_VERSION} build-whisper: docker-whisper ## Build the whisper container and zarf package @@ -134,7 +132,7 @@ build-whisper: docker-whisper ## Build the whisper container and zarf package docker-repeater: local-registry sdk-wheel ## Build the image (and tag it for the local registry) - docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} -f packages/repeater/Dockerfile . + docker build --platform=linux/${ARCH} --build-arg ARCH=${ARCH} --build-arg LOCAL_VERSION=${LOCAL_VERSION} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} -f packages/repeater/Dockerfile . docker tag ghcr.io/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} localhost:${REG_PORT}/defenseunicorns/leapfrogai/repeater:${LOCAL_VERSION} build-repeater: docker-repeater ## Build the repeater container and zarf package diff --git a/packages/api/Dockerfile b/packages/api/Dockerfile index 43cc7fc2b..1cd10d1fe 100644 --- a/packages/api/Dockerfile +++ b/packages/api/Dockerfile @@ -1,5 +1,6 @@ ARG ARCH -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder ARG SDK_DEST=src/leapfrogai_sdk/build diff --git a/packages/llama-cpp-python/Dockerfile b/packages/llama-cpp-python/Dockerfile index ced587646..63a55c2a0 100644 --- a/packages/llama-cpp-python/Dockerfile +++ b/packages/llama-cpp-python/Dockerfile @@ -1,5 +1,6 @@ ARG ARCH -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk # hardened and slim python w/ developer tools image FROM --platform=$BUILDPLATFORM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev as builder diff --git a/packages/repeater/Dockerfile b/packages/repeater/Dockerfile index 67c50e421..1dcca8f0a 100644 --- a/packages/repeater/Dockerfile +++ b/packages/repeater/Dockerfile @@ -1,5 +1,6 @@ ARG ARCH -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk # hardened and slim python w/ developer tools image FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder diff --git a/packages/text-embeddings/Dockerfile b/packages/text-embeddings/Dockerfile index 23371b67c..0e4d320e3 100644 --- a/packages/text-embeddings/Dockerfile +++ b/packages/text-embeddings/Dockerfile @@ -1,5 +1,6 @@ ARG ARCH -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk # hardened and slim python w/ developer tools image FROM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev-${ARCH} as builder diff --git a/packages/vllm/Dockerfile b/packages/vllm/Dockerfile index 52419b7ff..8485171e7 100755 --- a/packages/vllm/Dockerfile +++ b/packages/vllm/Dockerfile @@ -1,4 +1,5 @@ -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk FROM nvidia/cuda:12.2.2-devel-ubuntu22.04 as builder ARG SDK_DEST=src/leapfrogai_sdk/build diff --git a/packages/whisper/Dockerfile b/packages/whisper/Dockerfile index 24d06d536..7d04955fc 100644 --- a/packages/whisper/Dockerfile +++ b/packages/whisper/Dockerfile @@ -1,5 +1,6 @@ ARG ARCH -FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:latest as sdk +ARG LOCAL_VERSION +FROM ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${LOCAL_VERSION} as sdk FROM --platform=$BUILDPLATFORM ghcr.io/defenseunicorns/leapfrogai/python:3.11-dev as builder USER root diff --git a/uds-bundles/dev/README.md b/uds-bundles/dev/README.md index 06968b629..ae4a148cb 100644 --- a/uds-bundles/dev/README.md +++ b/uds-bundles/dev/README.md @@ -72,6 +72,8 @@ cd uds-bundles/dev/cpu uds create . ``` +Deploy a [UDS cluster](/README.md#uds) if one isn't deployed already + Deploy the LeapfrogAI bundle: ```shell uds deploy uds-bundle-leapfrogai*.tar.zst @@ -85,10 +87,13 @@ cd uds-bundles/dev/gpu uds create . ``` +Deploy a [UDS cluster](/README.md#uds) with the following flags, as so: + ```shell uds deploy {k3d-cluster-name} --set K3D_EXTRA_ARGS="--gpus=all --image=ghcr.io/justinthelaw/k3d-gpu-support:v1.27.4-k3s1-cuda" ``` + Deploy the LeapfrogAI bundle: ```shell uds deploy uds-bundle-leapfrogai-*.tar.zst --confirm From f11aa3d60134debcab92d687b7c739afccf453a2 Mon Sep 17 00:00:00 2001 From: Gavin Scallon Date: Tue, 9 Jul 2024 18:39:31 -0700 Subject: [PATCH 18/19] pull in changes from main and add macos to OS's which have been tested --- website/content/en/docs/local deploy guide/requirements.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/website/content/en/docs/local deploy guide/requirements.md b/website/content/en/docs/local deploy guide/requirements.md index 813c703a2..279ac1403 100644 --- a/website/content/en/docs/local deploy guide/requirements.md +++ b/website/content/en/docs/local deploy guide/requirements.md @@ -1,5 +1,5 @@ --- -title: Requirements +title: Requirements type: docs weight: 4 --- @@ -18,6 +18,7 @@ The following operating systems, hardware, architectures, and system specificati - 22.04.4 - 22.04.5 - Pop!_OS 22.04 LTS +- MacOS Sonoma 14.x / ARM64 (CPU-based deployments only) ### Hardware @@ -44,7 +45,7 @@ The following operating systems, hardware, architectures, and system specificati - For all "Isolated Network" installs, `wget`, `git` `clone` and `zarf package create` commands are assumed to have been completed prior to entering the isolated network. - For "Isolated Network" installs, ensure files and binaries from these commands are stored on a removable media device and subsequently uploaded to the isolated machine. - For specific tool versions, it is recommended to follow the "Isolated Network" instructions. - + ## System Requirements - Standard Unix-based operating system installed. @@ -65,7 +66,7 @@ Additional considerations are necessary for GPU deployments: - Presently, these instructions do not support time-slicing or configuring multi-instance GPU setups. - Over-scheduling GPU resources beyond their availability may result in the crash of backend pods. - To prevent crashing, install backends as CPU-only if all available GPU devices are already allocated. - + ## Additional User Information - All `cd` commands should be executed with respect to your project's working directory (PWD) within the development environment. Each new step should be considered as initiating from the root of that directory. From 6deffd5a6117d1b54481bd7772e51c98aff8ec18 Mon Sep 17 00:00:00 2001 From: Jon Perry Date: Fri, 12 Jul 2024 14:32:09 -0400 Subject: [PATCH 19/19] chore: update release workflow to set ARCh and LOCAL_VERSIOn docker build arguments --- .github/workflows/release.yaml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 996cceeb8..507a81210 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -40,15 +40,18 @@ jobs: with: python-version-file: 'pyproject.toml' - - name: Download Python Wheels - run: make sdk-wheel + - name: Download Python Wheels and Publish Builder Image + run: | + docker buildx build --platform amd64 --build-arg ARCH=amd64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${{ steps.get_version.outputs.version-without-v }} --push -f src/leapfrogai_sdk/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-sdk:${{ steps.get_version.outputs.version-without-v }} --push -f src/leapfrogai_sdk/Dockerfile . - name: Install Zarf uses: defenseunicorns/setup-zarf@f95763914e20e493bb5d45d63e30e17138f981d6 # v1.0.0 - name: Build and Publish API run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push -f packages/api/Dockerfile . + docker buildx build --platform amd64 --build-arg ARCH=amd64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push -f packages/api/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/leapfrogai-api:${{ steps.get_version.outputs.version-without-v }} --push -f packages/api/Dockerfile . docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/api-migrations:${{ steps.get_version.outputs.version-without-v }} --push -f Dockerfile.migrations --build-arg="MIGRATIONS_DIR=packages/api/supabase/migrations" . zarf package create packages/api --set=LEAPFROGAI_IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm @@ -86,7 +89,8 @@ jobs: - name: Build and Publish repeater run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push -f packages/repeater/Dockerfile . + docker buildx build --platform amd64 --build-arg ARCH=amd64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push -f packages/repeater/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/repeater:${{ steps.get_version.outputs.version-without-v }} --push -f packages/repeater/Dockerfile . zarf package create packages/repeater --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/repeater --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -99,7 +103,8 @@ jobs: - name: Build and Publish llama run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push -f packages/llama-cpp-python/Dockerfile . + docker buildx build --platform amd64 --build-arg ARCH=amd64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push -f packages/llama-cpp-python/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/llama-cpp-python:${{ steps.get_version.outputs.version-without-v }} --push -f packages/llama-cpp-python/Dockerfile . zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/llama-cpp-python --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -112,7 +117,7 @@ jobs: - name: Build and Publish vLLM run: | - docker buildx build -t ghcr.io/defenseunicorns/leapfrogai/vllm:${{ steps.get_version.outputs.version-without-v }} --push -f packages/vllm/Dockerfile . + docker buildx build --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/vllm:${{ steps.get_version.outputs.version-without-v }} --push -f packages/vllm/Dockerfile . zarf package create packages/vllm --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --confirm @@ -123,7 +128,8 @@ jobs: - name: Build and Publish Text-Embeddings run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ steps.get_version.outputs.version-without-v }} --push -f packages/text-embeddings/Dockerfile . + docker buildx build --platform amd64 --build-arg ARCH=amd64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ steps.get_version.outputs.version-without-v }} --push -f packages/text-embeddings/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/text-embeddings:${{ steps.get_version.outputs.version-without-v }} --push -f packages/text-embeddings/Dockerfile . zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/text-embeddings --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm @@ -136,7 +142,8 @@ jobs: - name: Build and Publish whisper run: | - docker buildx build --platform amd64,arm64 -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ steps.get_version.outputs.version-without-v }} --push -f packages/whisper/Dockerfile . + docker buildx build --platform amd64 --build-arg ARCH=amd64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ steps.get_version.outputs.version-without-v }} --push -f packages/whisper/Dockerfile . + docker buildx build --platform arm64 --build-arg ARCH=arm64 --build-arg LOCAL_VERSION=${{ steps.get_version.outputs.version-without-v }} -t ghcr.io/defenseunicorns/leapfrogai/whisper:${{ steps.get_version.outputs.version-without-v }} --push -f packages/whisper/Dockerfile . zarf package create packages/whisper --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture amd64 --confirm zarf package create packages/whisper --set=IMAGE_VERSION=${{ steps.get_version.outputs.version-without-v }} --architecture arm64 --confirm