From 8484dd47e09f38c2a204e61cb1fae0821ce7a1fe Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Sat, 21 Oct 2023 07:46:20 +0200 Subject: [PATCH 1/5] Add Dockerfile and example docker-compose.yml --- .dockerignore | 19 +++++++++++++++++++ .gitignore | 2 ++ Dockerfile | 20 ++++++++++++++++++++ docker-compose.yml | 41 +++++++++++++++++++++++++++++++++++++++++ entrypoint.sh | 9 +++++++++ 5 files changed, 91 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7ca43d3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.dockerignore +.gitignore +docker-compose.yml +Dockerfile +env/ +venv/ + +cache/ +/web/static/ +/web/media/ +*credentials* + +.idea/ +*.iml + +__pycache__/ +*.pyc + +data/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 66c044f..45e9a25 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ cache/ __pycache__/ *.pyc + +data/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9430084 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.11 +LABEL maintainer="Holger Bruch " + + +RUN apt-get install +RUN apt-get update && apt-get install -y \ + libpq-dev libgdal-dev libproj-dev libgeos-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY requirements.txt /app/ +COPY web/scrapers/ParkAPI2_sources/requirements.txt /app/web/scrapers/ParkAPI2_sources/ + +RUN pip install -r requirements.txt -r web/scrapers/ParkAPI2_sources/requirements.txt + +COPY . /app + +EXPOSE 8000 +ENTRYPOINT ["sh", "/app/entrypoint.sh"] +CMD runserver 0.0.0.0:8000 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4c879aa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,41 @@ +version: '3' +services: + db: + image: postgis/postgis:16-3.4 + volumes: + - ./data:/var/lib/postgresql/data + env_file: .env + ports: + - 5432:5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U parkapi2"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + api: + build: + context: . + dockerfile: ./Dockerfile + command: ["runserver", "0.0.0.0:8000"] + depends_on: + db: + condition: service_healthy + expose: + - 8000 + ports: + - 8000:8000 + env_file: .env + scraper: + build: + context: . + dockerfile: ./Dockerfile + command: + - pa_scrape + - scrape + - -j + - "100" + depends_on: + db: + condition: service_healthy + env_file: .env diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..ef8f6f3 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,9 @@ +# init the main database +cd /app/web +./manage.py migrate + +# don't create an admin interface per default +#./manage.py createsuperuser + +# start the server +./manage.py $@ From 6e6ae67683e8b2c841fb88a62b699d7b53693564 Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Sat, 21 Oct 2023 07:47:09 +0200 Subject: [PATCH 2/5] Add workflow to build image via action --- .github/workflows/build-and-publish.yaml | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/build-and-publish.yaml diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml new file mode 100644 index 0000000..ec5cdac --- /dev/null +++ b/.github/workflows/build-and-publish.yaml @@ -0,0 +1,52 @@ +name: build & publish Docker image + +on: + push: + branches: + # TODO: rename master to main + - master + # TODO: remove before merge + - docker + +jobs: + build-publish: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: set up Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: log into the GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: lowercase repo name + id: lower-repo + uses: ASzc/change-string-case-action@v6 + with: + string: ${{ github.repository }} + + - name: set current date as env variable + id: version + run: | + echo "builddate=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: build and push Docker image + uses: docker/build-push-action@v4 + with: + file: Dockerfile + push: true + tags: | + ghcr.io/${{ steps.lower-repo.outputs.lowercase }}:latest,ghcr.io/${{ steps.lower-repo.outputs.lowercase }}:${{ steps.version.outputs.builddate }} + platforms: linux/amd64,linux/arm64 + # https://docs.docker.com/build/ci/github-actions/cache/#cache-backend-api + cache-from: type=gha + cache-to: type=gha,mode=max + From 2e74e87d3a7848a60f966f532d0adb9b6c193259 Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Mon, 30 Oct 2023 19:30:16 +0100 Subject: [PATCH 3/5] make migration optional --- Dockerfile | 1 + docker-compose.yml | 2 ++ entrypoint.sh | 9 ++++++--- 3 files changed, 9 insertions(+), 3 deletions(-) mode change 100644 => 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 9430084..2ae091a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.11 LABEL maintainer="Holger Bruch " +ENV RUN_MIGRATION=false RUN apt-get install RUN apt-get update && apt-get install -y \ diff --git a/docker-compose.yml b/docker-compose.yml index 4c879aa..e796364 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,6 +26,8 @@ services: ports: - 8000:8000 env_file: .env + environment: + - RUN_MIGRATION=1 scraper: build: context: . diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 index ef8f6f3..300e048 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,9 @@ -# init the main database -cd /app/web -./manage.py migrate +cd web + +# Run migration only if explicitly set via ENV +if [ "$RUN_MIGRATION" != "false" ]; then + ./manage.py migrate +fi # don't create an admin interface per default #./manage.py createsuperuser From 12593aafd6d755ff421e327a357bffa7c3594f21 Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Sun, 10 Dec 2023 15:59:18 +0100 Subject: [PATCH 4/5] add optional find_location on startup --- docker-compose.yml | 2 ++ entrypoint.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e796364..504b80c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,8 @@ services: env_file: .env environment: - RUN_MIGRATION=1 + - ASSIGN_LOCATIONS=1 + scraper: build: context: . diff --git a/entrypoint.sh b/entrypoint.sh index 300e048..7d6d1f0 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,6 +5,10 @@ if [ "$RUN_MIGRATION" != "false" ]; then ./manage.py migrate fi +if [ "$ASSING_LOCATIONS" != "false" ]; then + ./manage.py pa_find_locations +fi + # don't create an admin interface per default #./manage.py createsuperuser From f908a0a0b16c6191f5e08ca6cb5570de913f05f9 Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Sat, 21 Sep 2024 08:48:20 +0200 Subject: [PATCH 5/5] incorporate review i.e. * switch to prebuilt gdal image * multistaged docker build * allow superuser creation via flag * use bash for script execution * set -euo pipefial for entrypoint.sh --- .dockerignore | 1 + Dockerfile | 35 ++++++++++++++++++++++++----------- entrypoint.sh | 14 +++++++++++--- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.dockerignore b/.dockerignore index 7ca43d3..a9f05c6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .dockerignore .gitignore +.env docker-compose.yml Dockerfile env/ diff --git a/Dockerfile b/Dockerfile index 2ae091a..c33dcc1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,34 @@ -FROM python:3.11 +FROM ghcr.io/osgeo/gdal:alpine-small-3.9.2 AS builder LABEL maintainer="Holger Bruch " -ENV RUN_MIGRATION=false +RUN apk add --no-cache build-base python3-dev py3-pip postgresql-client libpq-dev geos-dev -RUN apt-get install -RUN apt-get update && apt-get install -y \ - libpq-dev libgdal-dev libproj-dev libgeos-dev \ - && rm -rf /var/lib/apt/lists/* + +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" WORKDIR /app -COPY requirements.txt /app/ -COPY web/scrapers/ParkAPI2_sources/requirements.txt /app/web/scrapers/ParkAPI2_sources/ +COPY requirements.txt . +COPY web/scrapers/ParkAPI2_sources/requirements.txt ./ParkAPI2_sources_requirements.txt + +RUN pip install -r requirements.txt -r ParkAPI2_sources_requirements.txt + +# Operational stage +FROM ghcr.io/osgeo/gdal:alpine-small-3.9.2 +LABEL maintainer="Holger Bruch " + +RUN apk add --no-cache bash git python3 py3-pip postgresql-client geos -RUN pip install -r requirements.txt -r web/scrapers/ParkAPI2_sources/requirements.txt +COPY --from=builder /opt/venv /opt/venv +ENV RUN_MIGRATION=0 \ + CREATE_SUPERUSER=0 \ + ASSIGN_LOCATIONS=0 \ + PYTHONBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" -COPY . /app +WORKDIR /app +COPY . /app/ EXPOSE 8000 -ENTRYPOINT ["sh", "/app/entrypoint.sh"] +ENTRYPOINT ["/bin/bash", "/app/entrypoint.sh"] CMD runserver 0.0.0.0:8000 diff --git a/entrypoint.sh b/entrypoint.sh index 7d6d1f0..b44794f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,16 +1,24 @@ +#!/bin/bash +set -euo pipefail + cd web +if [ "$RUN_MIGRATION" -eq "1" ]; then + echo "RUN MIGRATION is TRUE" +fi # Run migration only if explicitly set via ENV -if [ "$RUN_MIGRATION" != "false" ]; then +if [ "$RUN_MIGRATION" -eq "1" ]; then ./manage.py migrate fi -if [ "$ASSING_LOCATIONS" != "false" ]; then +if [ "$ASSIGN_LOCATIONS" -eq "1" ]; then ./manage.py pa_find_locations fi # don't create an admin interface per default -#./manage.py createsuperuser +if [ "$CREATE_SUPERUSER" -eq "1" ]; then + ./manage.py createsuperuser +fi # start the server ./manage.py $@