From e13f76e93481a442509c6514d44973fe61fc7e33 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 14 Feb 2024 21:55:01 -0800 Subject: [PATCH 1/9] containerize gantry and move from spack to requirements.txt --- .envrc | 13 ++----------- .gitignore | 2 -- Dockerfile | 23 +++++++++++++++++++++++ docs/deploy.md | 28 ++++++++++++++++++++++++++++ pyproject.toml | 5 +++++ requirements.txt | 8 ++++++++ spack.yaml | 14 -------------- 7 files changed, 66 insertions(+), 27 deletions(-) create mode 100644 Dockerfile create mode 100644 docs/deploy.md create mode 100644 requirements.txt delete mode 100644 spack.yaml diff --git a/.envrc b/.envrc index dcea6a5..5646e6b 100644 --- a/.envrc +++ b/.envrc @@ -1,17 +1,8 @@ -#------------------------------------------------------------------------ -# Load Development Spack Environment (If Spack is installed.) -# -# Run 'direnv allow' from within the cloned repository to automatically -# load the spack environment when you enter the directory. -#------------------------------------------------------------------------ -if type spack &>/dev/null; then - . $SPACK_ROOT/share/spack/setup-env.sh - spack env activate -d . -fi +source $HOME/.venvs/gantry/bin/activate #------------------------------------------------------------------------ # Load Environment Variables from .env (if files exists) #------------------------------------------------------------------------ if [ -e .env ]; then source .env -fi +fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 372e265..85da267 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ __pycache__ .env -spack.lock -.spack-env db/*.db diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..18f1642 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# from https://github.com/GoogleContainerTools/distroless/blob/main/examples/python3-requirements/Dockerfile + +# Build a virtualenv using the appropriate Debian release +# * Install python3-venv for the built-in Python3 venv module (not installed by default) +# * Install gcc libpython3-dev to compile C Python modules +# * In the virtualenv: Update pip setuputils and wheel to support building new packages +FROM debian:12-slim AS build +RUN apt-get update && \ + apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ + python3 -m venv /venv && \ + /venv/bin/pip install --upgrade pip setuptools wheel + +# Build the virtualenv as a separate step: Only re-execute this step when requirements.txt changes +FROM build AS build-venv +COPY requirements.txt /requirements.txt +RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt + +# Copy the virtualenv into a distroless image +FROM gcr.io/distroless/python3-debian12:nonroot +COPY --from=build-venv /venv /venv +COPY ./gantry /app/gantry +WORKDIR /app +ENTRYPOINT ["/venv/bin/python", "-m", "gantry"] \ No newline at end of file diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..5634c02 --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,28 @@ +# Deploy + +Gantry is distributed via containers. You can build an image using your favorite container engine: + +```bash +podman build -t gantry . +``` + +In order for the application to run, you'll need to supply two things to the image: environment variables and a database file. + +When running locally, you can get by with supplying an env file and volume, like so: + +```bash +podman run -it -p 8080:8080 --env-file .env -v DB_PATH:/db gantry +``` + +When running Gantry within Kubernetes, you could use [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/). The only requirement is that the database should exist outside the container for backup and persistence purposes. + +## Environment + +The following variables should be exposed to the container. Those **bolded** are required and do not have defaults set in the application. + +- **`PROMETHEUS_URL`** - should end in `/api/v1` +- `PROMETHEUS_COOKIE` - only needed when Prometheus requires authentication +- **`GITLAB_URL`** - should end in the endpoint for the Spack project API: `/api/v4/projects/2` +- **`GITLAB_API_TOKEN`** - this token should have API read access +- **`GITLAB_WEBHOOK_TOKEN`** - coordinate this value with the collection webhook +- **`DB_FILE`** - path where the application can access the SQLite file \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4620ee1..3804a67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,3 +2,8 @@ profile = "black" skip_gitignore = true color_output = true + +dependencies = [ + "aiohttp", + "aiosqlite" +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..66434ce --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +aiohttp==3.9.3 +aiosignal==1.3.1 +aiosqlite==0.19.0 +attrs==23.2.0 +frozenlist==1.4.1 +idna==3.6 +multidict==6.0.5 +yarl==1.9.4 \ No newline at end of file diff --git a/spack.yaml b/spack.yaml deleted file mode 100644 index 44863c0..0000000 --- a/spack.yaml +++ /dev/null @@ -1,14 +0,0 @@ -spack: - specs: - - python - - py-aiohttp - - py-pytest - - py-pytest-asyncio - - py-flake8 - - py-black - - py-isort - - py-aiosqlite - - sqlite - view: true - concretizer: - unify: true From bea952c1584d0ec58c3069e8379e8c5258cfa5a8 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 16 Feb 2024 11:25:17 -0800 Subject: [PATCH 2/9] virtualenv loading is not user-agnostic --- .envrc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.envrc b/.envrc index 5646e6b..0745c70 100644 --- a/.envrc +++ b/.envrc @@ -1,8 +1,6 @@ -source $HOME/.venvs/gantry/bin/activate - #------------------------------------------------------------------------ # Load Environment Variables from .env (if files exists) #------------------------------------------------------------------------ if [ -e .env ]; then source .env -fi \ No newline at end of file +fi From 8af58701c8f89a55bbddd157da5701bee2953d62 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Fri, 16 Feb 2024 11:31:08 -0800 Subject: [PATCH 3/9] add a few more env variables to docs --- docs/deploy.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index 5634c02..92ff1ee 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -25,4 +25,7 @@ The following variables should be exposed to the container. Those **bolded** are - **`GITLAB_URL`** - should end in the endpoint for the Spack project API: `/api/v4/projects/2` - **`GITLAB_API_TOKEN`** - this token should have API read access - **`GITLAB_WEBHOOK_TOKEN`** - coordinate this value with the collection webhook -- **`DB_FILE`** - path where the application can access the SQLite file \ No newline at end of file +- **`DB_FILE`** - path where the application can access the SQLite file +- `MAX_GET_SIZE` - the maximum `GET` request (in bytes), default is 8MB +- `GANTRY_HOST` - web app hostname, default is `localhost` +- `GANTRY_PORT` - web app port, default is `8080` \ No newline at end of file From c1c5f435c8705a9a849d2bc9f08f741f09f65a78 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:27:49 -0800 Subject: [PATCH 4/9] adding spack environment back --- spack.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 spack.yaml diff --git a/spack.yaml b/spack.yaml new file mode 100644 index 0000000..44863c0 --- /dev/null +++ b/spack.yaml @@ -0,0 +1,14 @@ +spack: + specs: + - python + - py-aiohttp + - py-pytest + - py-pytest-asyncio + - py-flake8 + - py-black + - py-isort + - py-aiosqlite + - sqlite + view: true + concretizer: + unify: true From 8341a7882564ff45797adce275dd36e94bbd1ade Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:29:14 -0800 Subject: [PATCH 5/9] preserve pyproject.toml changes for another PR --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3804a67..4620ee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,3 @@ profile = "black" skip_gitignore = true color_output = true - -dependencies = [ - "aiohttp", - "aiosqlite" -] \ No newline at end of file From 4ee0db44526f1bc95b03a2e6c2b7cc1f9d340332 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 22 Feb 2024 14:26:42 -0800 Subject: [PATCH 6/9] fix: Dockerfile prune unnecessary FROM --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 18f1642..3c8100b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,15 +9,12 @@ RUN apt-get update && \ apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \ python3 -m venv /venv && \ /venv/bin/pip install --upgrade pip setuptools wheel - -# Build the virtualenv as a separate step: Only re-execute this step when requirements.txt changes -FROM build AS build-venv COPY requirements.txt /requirements.txt RUN /venv/bin/pip install --disable-pip-version-check -r /requirements.txt # Copy the virtualenv into a distroless image FROM gcr.io/distroless/python3-debian12:nonroot -COPY --from=build-venv /venv /venv +COPY --from=build /venv /venv COPY ./gantry /app/gantry WORKDIR /app -ENTRYPOINT ["/venv/bin/python", "-m", "gantry"] \ No newline at end of file +ENTRYPOINT ["/venv/bin/python", "-m", "gantry"] From 5f66e6aaaaa545a044cbfcae2401bcf11068864d Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 22 Feb 2024 14:30:04 -0800 Subject: [PATCH 7/9] doc updates --- docs/deploy.md | 6 ++++-- docs/home.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 92ff1ee..d326fc5 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -11,9 +11,11 @@ In order for the application to run, you'll need to supply two things to the ima When running locally, you can get by with supplying an env file and volume, like so: ```bash -podman run -it -p 8080:8080 --env-file .env -v DB_PATH:/db gantry +podman run -it -p 8080:8080 --env-file .env -v LOCAL_DB_PATH:DB_FILE gantry ``` +Where `LOCAL_DB_PATH` is the absolute path to the database file on your local system. `DB_FILE` is where you would like the application to access the database. Make sure this lines up with your environment. + When running Gantry within Kubernetes, you could use [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/). The only requirement is that the database should exist outside the container for backup and persistence purposes. ## Environment @@ -28,4 +30,4 @@ The following variables should be exposed to the container. Those **bolded** are - **`DB_FILE`** - path where the application can access the SQLite file - `MAX_GET_SIZE` - the maximum `GET` request (in bytes), default is 8MB - `GANTRY_HOST` - web app hostname, default is `localhost` -- `GANTRY_PORT` - web app port, default is `8080` \ No newline at end of file +- `GANTRY_PORT` - web app port, default is `8080` diff --git a/docs/home.md b/docs/home.md index c598cdf..336b325 100644 --- a/docs/home.md +++ b/docs/home.md @@ -8,4 +8,4 @@ 2. [Data Collection](data-collection.md) 3. [Architecture](arch.md) 4. [Prediction](prediction.md) - +5. [Deployment](deploy.md) From f54656223688f4be216d69fea5493150f810c77b Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 22 Feb 2024 14:46:06 -0800 Subject: [PATCH 8/9] revert spack changes fully --- .envrc | 11 +++++++++++ .gitignore | 2 ++ 2 files changed, 13 insertions(+) diff --git a/.envrc b/.envrc index 0745c70..dcea6a5 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,14 @@ +#------------------------------------------------------------------------ +# Load Development Spack Environment (If Spack is installed.) +# +# Run 'direnv allow' from within the cloned repository to automatically +# load the spack environment when you enter the directory. +#------------------------------------------------------------------------ +if type spack &>/dev/null; then + . $SPACK_ROOT/share/spack/setup-env.sh + spack env activate -d . +fi + #------------------------------------------------------------------------ # Load Environment Variables from .env (if files exists) #------------------------------------------------------------------------ diff --git a/.gitignore b/.gitignore index 85da267..e152a6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ __pycache__ .env db/*.db +spack.lock +.spack-env From 626fddd201c568e398b2cd784e7f1a11f4e84324 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 22 Feb 2024 14:52:19 -0800 Subject: [PATCH 9/9] make .gitignore changes null --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e152a6e..372e265 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ __pycache__ .env -db/*.db spack.lock .spack-env +db/*.db