diff --git a/.dockerignore b/.dockerignore index e3fede1b2..19338251b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,12 +15,12 @@ /Vagrantfile.local /npm-debug.log /tmp/ +/.venv/ /venv/ __pycache__ coverage node_modules/ pytest.ini -requirements-gae.txt # Distribution / packaging *.egg @@ -41,10 +41,9 @@ sdist/ var/ wheels/ -.gcloudignore .git +.github/ .gitignore /*initdb.d/ Makefile -app.yml -docker-compose*.yml \ No newline at end of file +docker-compose*.yml diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 464aef1c5..d05991bde 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -18,10 +18,6 @@ jobs: steps: - uses: actions/checkout@v3 - run: make test - - uses: actions/upload-artifact@v3 - with: - name: django-coverage-report - path: htmlcov selenium-tests: runs-on: ubuntu-latest @@ -30,8 +26,3 @@ jobs: steps: - uses: actions/checkout@v3 - run: make selenium-test - - uses: actions/upload-artifact@v3 - with: - name: django-coverage-report - path: htmlcov - diff --git a/Dockerfile b/Dockerfile index b8c724cf4..873d7fc44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,56 +1,58 @@ -# Use an official Python runtime based on Debian 10 "buster" as a parent image. -FROM python:3.8.1-slim-buster - -# Add user that will be used in the container. -RUN useradd wagtail - -# Port used by this container to serve HTTP. -EXPOSE 8000 - -# Set environment variables. -# 1. Force Python stdout and stderr streams to be unbuffered. -# 2. Set PORT variable that is used by Gunicorn. This should match "EXPOSE" -# command. -ENV PYTHONUNBUFFERED=1 \ - PORT=8000 - -COPY requirements.txt / - -# Install system packages required by Wagtail and Django. +FROM python:3.8.17-slim-bookworm AS builder +ENV PIP_NO_CACHE_DIR=1 \ + PYTHONUNBUFFERED=1 RUN apt-get update --yes --quiet \ - && apt-get install --yes --quiet --no-install-recommends \ + && apt-get install --yes --quiet --no-install-recommends \ build-essential \ gettext \ git \ - libpq5 \ + libpango1.0-0 \ libpq-dev \ + libpq5 \ + && pip install --upgrade pip \ + && pip install pip-tools \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /opt +RUN python -m venv venv +ENV PATH="/opt/venv/bin:$PATH" +RUN pip install --upgrade pip +ARG requirements=requirements.txt +COPY ${requirements} . +RUN pip install -r $requirements + +FROM python:3.8.17-slim-bookworm AS base +EXPOSE 8000 +ENV PATH="/opt/venv/bin:$PATH" \ + PIP_NO_CACHE_DIR=1 \ + PYTHONUNBUFFERED=1 +RUN useradd wagtail +RUN apt-get update --yes --quiet \ + && apt-get install --yes --quiet --no-install-recommends \ + gettext \ libpango1.0-0 \ - && pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir "gunicorn==20.0.4" \ - && pip install --no-cache-dir -r /requirements.txt \ - && apt remove --yes --quiet git build-essential libpq-dev \ - && apt autoremove --yes --quiet \ + libpq5 \ && rm -rf /var/lib/apt/lists/* - -# Use /app folder as a directory where the source code is stored. WORKDIR /app - -# Set this directory to be owned by the "wagtail" user. This Wagtail project -# uses SQLite, the folder needs to be owned by the user that -# will be writing to the database file. RUN chown wagtail:wagtail /app -# Copy the source code of the project into the container. +FROM base AS dev +RUN apt-get update --yes --quiet \ + && apt-get install --yes --quiet --no-install-recommends \ + tini \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder --chown=wagtail:wagtail /opt/venv /opt/venv COPY --chown=wagtail:wagtail . . - -# Use user "wagtail" to run the build commands below and the server itself. USER wagtail +ENTRYPOINT ["tini", "--"] +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] -# Collect static files. +FROM base AS prod +# PORT variable is used by Gunicorn. Should match "EXPOSE" command. +ENV PORT=8000 +USER wagtail +COPY --from=builder --chown=wagtail:wagtail /opt/venv /opt/venv +RUN pip install "gunicorn==20.0.4" +COPY --chown=wagtail:wagtail . . RUN python manage.py collectstatic --noinput --clear - -# Compile files for localization RUN python manage.py compilemessages - -# Start the application server. -CMD gunicorn iogt.wsgi:application +CMD ["gunicorn", "iogt.wsgi:application"] diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 2b0e1e0de..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,14 +0,0 @@ -FROM python:3.8.1-buster - -ENV PYTHONUNBUFFERED 1 - -WORKDIR /app - -RUN apt-get update --yes --quiet \ - && apt-get install --yes --quiet --no-install-recommends gettext tini \ - && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt requirements.dev.txt / -RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir pip-tools \ - && pip install --no-cache-dir -r /requirements.dev.txt diff --git a/Makefile b/Makefile index 06d6d81e4..843b4eb1f 100644 --- a/Makefile +++ b/Makefile @@ -12,20 +12,16 @@ migrate: setup: make migrate make update_elasticsearch_index -# SSH into the django container -ssh: - docker-compose run django /bin/bash up: docker-compose up update_elasticsearch_index: docker-compose run django python manage.py update_index compile-requirements: - docker-compose run --rm django /app/scripts/compile-requirements.sh + docker-compose -f docker-compose.builder.yml run --rm builder test: - docker-compose -f docker-compose.test.yml up --build -d django + docker-compose -f docker-compose.test.yml up --build -d docker-compose -f docker-compose.test.yml exec -T django python manage.py collectstatic --noinput - docker-compose -f docker-compose.test.yml exec -T django coverage run --source='.' manage.py test --noinput --parallel $(IOGT_TEST_PARALLEL) - docker-compose -f docker-compose.test.yml exec -T django coverage html + docker-compose -f docker-compose.test.yml exec -T django python manage.py test --noinput --parallel $(IOGT_TEST_PARALLEL) docker-compose -f docker-compose.test.yml down --remove-orphans selenium-test: selenium-up selenium-local selenium-down selenium-up: diff --git a/docker-compose.builder.yml b/docker-compose.builder.yml new file mode 100644 index 000000000..8ccae5d40 --- /dev/null +++ b/docker-compose.builder.yml @@ -0,0 +1,11 @@ +version: "3" + +services: + builder: + build: + context: . + target: builder + command: + - /app/scripts/compile-requirements.sh + volumes: + - ./:/app/ diff --git a/docker-compose.selenium.yml b/docker-compose.selenium.yml index 33bc9c4de..8244b6449 100644 --- a/docker-compose.selenium.yml +++ b/docker-compose.selenium.yml @@ -1,15 +1,15 @@ -version: '3.5' +version: "3" services: django: build: context: ./ - dockerfile: Dockerfile.dev + target: dev + args: + requirements: requirements.dev.txt environment: DJANGO_SETTINGS_MODULE: iogt.settings.test command: ["tini", "--", "sleep", "infinity"] - volumes: - - ./:/app/ depends_on: - database - selenium-hub diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 1b96e7fc0..abf5e1f4e 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,10 +1,12 @@ -version: '3.5' +version: "3" services: django: build: context: ./ - dockerfile: Dockerfile.dev + target: dev + args: + requirements: requirements.dev.txt environment: DB_HOST: db DB_NAME: iogt @@ -12,13 +14,10 @@ services: DB_PORT: '5432' DB_USER: iogt DJANGO_SETTINGS_MODULE: iogt.settings.test - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - ./:/app/ depends_on: - db db: - image: postgres:alpine + image: postgres:14-alpine environment: POSTGRES_USER: iogt POSTGRES_PASSWORD: iogt diff --git a/docker-compose.yml b/docker-compose.yml index 5feeea4e0..51a436a76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,11 @@ services: django: build: context: ./ - dockerfile: Dockerfile.dev + target: dev + args: + requirements: requirements.dev.txt environment: DJANGO_SETTINGS_MODULE: iogt.settings.dev - command: python manage.py runserver 0.0.0.0:8000 volumes: - ./:/app/ ports: diff --git a/docs/dependency-management.md b/docs/dependency-management.md index 185692c15..041bd5594 100644 --- a/docs/dependency-management.md +++ b/docs/dependency-management.md @@ -1,37 +1,50 @@ # Introduce pip-tools ## What is it? -Pip-tools is a command-line toolset for managing dependencies in Python projects. -It streamlines the process of handling project dependencies by providing essential functionalities. +Pip-tools is a command-line toolset for managing dependencies in Python projects. It streamlines the process of handling project dependencies by providing essential functionalities. ## Why do we use it? ### Dependency Management -Pip-tools helps manage a project's dependencies by resolving and pinning specific versions. -It ensures that everyone working on the project uses the same versions of dependencies, promoting consistency and reproducibility. + +Pip-tools helps manage a project's dependencies by resolving and pinning specific versions. It ensures that everyone working on the project uses the same versions of dependencies, promoting consistency and reproducibility. ### Simplified Workflow -Pip-tools simplifies the workflow of managing dependencies by keeping the requirements.in file separate from the requirements.txt file. + The requirements.in file contains high-level dependencies, while the requirements.txt file contains the pinned versions. This separation enables easy updating and maintenance of dependencies. ## How do we use it? -### For Virtual Environment: +### For virtual environment: -#### Install or Upgrade pip: -```pip install --upgrade pip``` +Install or upgrade pip. -#### Install pip-tools: -```pip install pip-tools``` +```sh +pip install --upgrade pip +``` -#### Generate Pinned Versions: -Add or update the package and its version specifier according to your needs in .in file. +Install pip-tools. -```pip-compile --generate-hashes --resolver=backtracking -o .txt .in``` +```sh +pip install pip-tools +``` + +To generate pinned versions, add or update a package and its version specifier according to your needs in the _requirements.in_ or _requirements.dev.in_ file. + +```sh +pip-compile --generate-hashes --resolver=backtracking -o .txt .in +``` ### For Docker -Add or update the package and its version specifier according to your needs in .in file. + +Add or update the package and its version specifier according to your needs in a _\*.in_ file. In the project root directory. ``` make compile-requirements -``` \ No newline at end of file +``` + +Alternatively, you could run `pip-compile` directly in the _builder_ container. + +``` +docker compose -f docker-compose.builder.yml run --rm builder pip-compile ... +``` diff --git a/requirements.dev.in b/requirements.dev.in index a03253f00..9902fff74 100644 --- a/requirements.dev.in +++ b/requirements.dev.in @@ -1,5 +1,4 @@ -r requirements.txt -beautifulsoup4 coverage django-test-plus>=1.4.0 factory-boy==3.2.* diff --git a/requirements.dev.txt b/requirements.dev.txt index 83ad563e4..fb6acd5f8 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -58,7 +58,6 @@ beautifulsoup4==4.9.3 \ --hash=sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25 \ --hash=sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666 # via - # -r /app/requirements.dev.in # -r /app/requirements.txt # django-google-analytics-app # wagtail diff --git a/requirements.in b/requirements.in index 1d2948065..bc930540c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ Markdown==3.3.7 +beautifulsoup4~=4.9.3 cairosvg==2.7.0 django-allauth~=0.44 django-comments-xtd==2.9.* diff --git a/requirements.txt b/requirements.txt index 0a4b74624..811f0af04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ beautifulsoup4==4.9.3 \ --hash=sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25 \ --hash=sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666 # via + # -r /app/requirements.in # django-google-analytics-app # wagtail billiard==4.1.0 \ diff --git a/scripts/compile-requirements.sh b/scripts/compile-requirements.sh index b272e0c1c..b7db3ac30 100755 --- a/scripts/compile-requirements.sh +++ b/scripts/compile-requirements.sh @@ -6,5 +6,10 @@ REQ_FILES=( ) for f in "${REQ_FILES[@]}"; do - pip-compile --generate-hashes --resolver=backtracking -o ${f}.txt ${f}.in || exit 1; + pip-compile --cache-dir=/tmp/pip-tools \ + --generate-hashes \ + --resolver=backtracking \ + --output-file ${f}.txt \ + ${f}.in \ + || exit 1; done