From 4cee1c65d8728503ca81fc6cf46de4a2200ada5f Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 21 Jun 2023 04:37:39 +0530 Subject: [PATCH 01/45] New basic and KISS Dockerfile for funnel --- .dockerignore | 2 +- .gitignore | 2 - .pre-commit-config.yaml | 8 +-- Dockerfile | 139 +++++++++------------------------------- sample.env | 8 ++- 5 files changed, 41 insertions(+), 118 deletions(-) diff --git a/.dockerignore b/.dockerignore index d8b7b6217..58e88bd14 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,7 +9,7 @@ **/.webassets-cache **/__pycache__ .webpack_cache -.cache +.eslintcache # MacOS file **/.DS_Store diff --git a/.gitignore b/.gitignore index 2c16442e0..14e938068 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,6 @@ __pycache__ .nox monkeytype.sqlite3 -.ci-cache -.cache # MacOS file .DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8c84b2c7..89addc6bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -194,10 +194,10 @@ repos: # hooks: # - id: dockerfile_lint # files: .*Dockerfile.* - # - repo: https://github.com/hadolint/hadolint - # rev: v2.12.1-beta - # hooks: - # - id: hadolint-docker + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint-docker - repo: https://github.com/IamTheFij/docker-pre-commit rev: v3.0.1 hooks: diff --git a/Dockerfile b/Dockerfile index 9693027e2..9a5378de9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,112 +1,31 @@ # syntax=docker/dockerfile:1.4 -FROM nikolaik/python-nodejs:python3.11-nodejs20-bullseye as base - -# https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile -# https://stackoverflow.com/questions/68465355/what-is-the-meaning-of-set-o-pipefail-in-bash-script -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -LABEL Name=Funnel -LABEL Version=0.1 - -USER pn -RUN \ - mkdir -pv /home/pn/.cache/pip /home/pn/.npm /home/pn/tmp /home/pn/app /home/pn/app/coverage && \ - chown -R pn:pn /home/pn/.cache /home/pn/.npm /home/pn/tmp /home/pn/app /home/pn/app/coverage -EXPOSE 3000 -WORKDIR /home/pn/app - -ENV PATH "$PATH:/home/pn/.local/bin" - -FROM base as devtest_base -USER root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -yqq && \ - apt-get install -yqq --no-install-recommends lsb-release && \ - sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ - apt-get update -yqq && apt-get upgrade -yqq && \ - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends firefox-esr postgresql-client-15 && \ - cd /tmp/ && \ - curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest | grep browser_download_url | grep 'linux64.tar.gz\"'| grep -o 'http.*\.gz') > gecko.tar.gz && \ - tar -xvzf gecko.tar.gz && \ - rm gecko.tar.gz && \ - chmod +x geckodriver && \ - mv geckodriver /usr/local/bin && \ - apt-get autoclean -yqq && \ - apt-get autoremove -yqq && \ - cd /home/pn/app -USER pn - -FROM base as assets -COPY --chown=pn:pn package.json package.json -COPY --chown=pn:pn package-lock.json package-lock.json -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm ci -COPY --chown=pn:pn ./funnel/assets ./funnel/assets -COPY --chown=pn:pn .eslintrc.js .eslintrc.js -COPY --chown=pn:pn webpack.config.js webpack.config.js -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm run build - -FROM base as dev_assets -COPY --chown=pn:pn package.json package.json -COPY --chown=pn:pn package-lock.json package-lock.json -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npm install -COPY --chown=pn:pn ./funnel/assets ./funnel/assets -COPY --chown=pn:pn .eslintrc.js .eslintrc.js -COPY --chown=pn:pn webpack.config.js webpack.config.js -RUN --mount=type=cache,target=/root/.npm \ - --mount=type=cache,target=/home/pn/.npm,uid=1000,gid=1000 npx webpack --mode development --progress - -FROM base as deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements/base.txt requirements/base.txt -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 \ - pip install --upgrade pip && \ - pip install --use-pep517 -r requirements/base.txt - -FROM devtest_base as test_deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements/base.txt requirements/base.txt -COPY --chown=pn:pn requirements/test.txt requirements/test.txt -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 pip install --use-pep517 -r requirements/test.txt - -FROM devtest_base as dev_deps -COPY --chown=pn:pn Makefile Makefile -RUN make deps-editable -COPY --chown=pn:pn requirements requirements -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 pip install --use-pep517 -r requirements/dev.txt -COPY --from=dev_assets --chown=pn:pn /home/pn/app/node_modules /home/pn/app/node_modules - -FROM deps as production -COPY --chown=pn:pn . . -COPY --chown=pn:pn --from=assets /home/pn/app/funnel/static /home/pn/app/funnel/static -ENTRYPOINT ["uwsgi", "--ini"] - -FROM production as supervisor -USER root -RUN \ - apt-get update -yqq && \ - apt-get install -yqq --no-install-recommends supervisor && \ - apt-get autoclean -yqq && \ - apt-get autoremove -yqq && \ - mkdir -pv /var/log/supervisor -COPY ./docker/supervisord/supervisord.conf /etc/supervisor/supervisord.conf -# COPY ./docker/uwsgi/emperor.ini /etc/uwsgi/emperor.ini -ENTRYPOINT ["/usr/bin/supervisord"] - -FROM test_deps as test -ENV PWD=/home/pn/app -COPY --chown=pn:pn . . - -COPY --chown=pn:pn --from=assets /home/pn/app/funnel/static /home/pn/app/funnel/static -ENTRYPOINT ["/home/pn/app/docker/entrypoints/ci-test.sh"] -FROM dev_deps as dev -RUN --mount=type=cache,target=/home/pn/.cache/pip,uid=1000,gid=1000 cp -R /home/pn/.cache/pip /home/pn/tmp/.cache_pip -RUN mv /home/pn/tmp/.cache_pip /home/pn/.cache/pip -COPY --chown=pn:pn --from=dev_assets /home/pn/app/funnel/static /home/pn/app/funnel/static +FROM node:lts-alpine as assets +USER node +WORKDIR /home/node/app +RUN mkdir -pv /home/node/app/funnel/static/build /home/node/app/funnel/static/build_cache +COPY --chown=node:node package.json package-lock.json ./ +RUN --mount=type=cache,target=/home/node/.npm/,uid=1000,gid=1000 npm ci +COPY --chown=node:node ./funnel/assets/ ./funnel/assets/ +COPY --chown=node:node webpack.config.js .eslintrc.js ./ +RUN --mount=type=cache,target=/home/node/app/.webpack_cache/,uid=1000,gid=1000 \ + --mount=type=cache,target=/home/node/app/funnel/static/build_cache/,uid=1000,gid=1000 \ + cp -R funnel/static/build_cache funnel/static/build \ + && npm run build \ + && cp -R funnel/static/build funnel/static/build_cache \ + && cp -R funnel/static/build funnel/static/built + +FROM python:3.11-bullseye as app +LABEL maintainer="Hasgeek" +RUN chsh -s /usr/sbin/nologin root +RUN addgroup --gid 1000 funnel && adduser --uid 1000 --gid 1000 funnel +ENV PATH "$PATH:/home/funnel/.local/bin" +USER funnel +WORKDIR /home/funnel/app + +COPY --chown=funnel:funnel Makefile Makefile +COPY --chown=funnel:funnel requirements/base.txt requirements/base.txt +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python + +COPY --chown=funnel:funnel . . +COPY --from=assets --chown=funnel:funnel /home/node/app/funnel/static/built/ funnel/static/build diff --git a/sample.env b/sample.env index 1ace7597f..105452ec3 100644 --- a/sample.env +++ b/sample.env @@ -59,6 +59,7 @@ FLASK_LASTUSER_SECRET_KEYS='["make-this-something-random", "older-secret-keys-he # name property APP_FUNNEL_SITE_ID=hasgeek APP_SHORTLINK_SITE_ID=shortlink +APP_UNSUBSCRIBE_SITE_ID=unsubscribe # Some templates pick up site title from this setting, but other places do not as any # value here is not localizable for translations APP_FUNNEL_SITE_TITLE=Hasgeek @@ -110,7 +111,7 @@ FLASK_SES_NOTIFICATION_TOPICS='[]' # --- Logging # Logfile (will be logrotated by date) -FLASK_LOGFILE=error.log +FLASK_LOGFILE=logs/error.log # List of email addresses to send error reports with traceback and local var dump # This requires SMTP config (above) FLASK_ADMINS='["webmaster@example.com"]' @@ -120,6 +121,11 @@ FLASK_TELEGRAM_ERROR_CHATID=null FLASK_TELEGRAM_ERROR_APIKEY=null # Send error reports to Slack webhooks (multiple); config spec TODO FLASK_SLACK_LOGGING_WEBHOOKS=null +# Overrides for metrics being sent to telegraf +# STATSD_HOST=127.0.0.1 +# STATSD_PORT=8125 +# STATSD_IPV6=False +# STATSD_TAGS=False # --- Hasgeek app integrations # Imgee image server From 9adfd6179025b372e9dfd57102a111df57ecb2b1 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 21 Jun 2023 05:58:26 +0530 Subject: [PATCH 02/45] Install supervisor in Dockerfile --- Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9a5378de9..1dd941988 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,15 @@ RUN --mount=type=cache,target=/home/node/app/.webpack_cache/,uid=1000,gid=1000 \ FROM python:3.11-bullseye as app LABEL maintainer="Hasgeek" RUN chsh -s /usr/sbin/nologin root +# hadolint ignore=DL3008 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends supervisor \ + && apt-get autoclean -yqq \ + && apt-get autoremove -yqq \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -pv /var/log/supervisor +COPY ./docker/supervisord/supervisord.conf /etc/supervisor/supervisord.conf RUN addgroup --gid 1000 funnel && adduser --uid 1000 --gid 1000 funnel ENV PATH "$PATH:/home/funnel/.local/bin" USER funnel From 48f10b68a50eebcd20eac2915d8b8fd4f1a5ad0d Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 23 Jun 2023 12:46:03 +0530 Subject: [PATCH 03/45] Working funnel docker container using uwsgi in http mode --- Dockerfile | 2 + docker-compose.yml | 232 ++++++---------------------------- docker/entrypoints/dev.sh | 8 -- docker/entrypoints/migrate.sh | 8 ++ docker/uwsgi/funnel.ini | 6 +- docker/uwsgi/shortlink.ini | 12 ++ sample.env | 5 +- 7 files changed, 64 insertions(+), 209 deletions(-) delete mode 100755 docker/entrypoints/dev.sh create mode 100755 docker/entrypoints/migrate.sh create mode 100644 docker/uwsgi/shortlink.ini diff --git a/Dockerfile b/Dockerfile index 1dd941988..d270470e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,3 +38,5 @@ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make ins COPY --chown=funnel:funnel . . COPY --from=assets --chown=funnel:funnel /home/node/app/funnel/static/built/ funnel/static/build +RUN mkdir -pv /home/funnel/app/logs +ENTRYPOINT [ "uwsgi", "--ini" ] diff --git a/docker-compose.yml b/docker-compose.yml index 3c9b0d1d9..526e73e3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,210 +1,50 @@ name: funnel -x-postgres: &postgres - image: postgres:latest - restart: always - user: postgres - environment: - - POSTGRES_HOST_AUTH_METHOD=trust - - POSTGRES_USER=postgres - expose: - - 5432 - healthcheck: - interval: 5s - timeout: 5s - retries: 5 -x-redis: &redis - image: redis:latest - expose: - - 6379 - restart: always - healthcheck: - test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] -x-app: &app - extends: - file: docker/compose/services.yml - service: funnel-prod - build: - context: . - target: production - image: funnel - profiles: - - production - depends_on: - - redis - environment: - - REDIS_HOST=redis -x-test: &test-app - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-test - profiles: - - test - build: - context: . - dockerfile: ci.Dockerfile - working_dir: /home/pn/app - user: pn - volumes: - - ./.cache/.npm:/home/pn/.npm - - ./.cache/node_modules:/home/pn/app/node_modules - - ./.cache/pip:/home/pn/.cache/pip - - ./.cache/.local:/home/pn/.local - - ./coverage:/home/pn/app/coverage - restart: 'no' services: - app: - <<: *app - volumes: - - ./instance/settings.py:/home/pn/app/instance/settings.py - - ./docker/uwsgi/funnel.ini:/home/pn/funnel.ini:ro - command: ../funnel.ini + main: &main-app + image: funnel + build: + context: . + target: app + logging: + driver: json-file + options: + max-size: 200k + max-file: 10 + links: + - redis + depends_on: + - redis + env_file: + - .env + environment: + - REDIS_HOST=redis + - FLASK_RUN_HOST=host.docker.internal + command: ['docker/uwsgi/funnel.ini'] expose: - 6400 ports: - 6400:6400 - pre-test: - <<: *test-app - user: root - entrypoint: ['/home/pn/app/docker/entrypoints/ci-pre-test.sh'] - test: - <<: *test-app - depends_on: - pre-test: - condition: service_completed_successfully - redis-test: - condition: service_healthy - db-test: - condition: service_healthy - links: - - db-test - - redis-test - environment: - - REDIS_HOST=redis-test - - DB_HOST=db-test - - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db-test/funnel_testing - - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db-test/geoname_testing - db-test: - <<: *postgres - profiles: - - test - volumes: - - postgres_test:/var/lib/postgresql/data - - ./docker/initdb/test.sh:/docker-entrypoint-initdb.d/test.sh:ro - healthcheck: - test: ['CMD-SHELL', 'psql funnel_testing'] - redis-test: - <<: *redis - profiles: - - test - volumes: - - redis_test:/data - dev: - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-dev - container_name: funnel-dev - profiles: - - dev - - dev-no-watch - build: - context: . - target: dev - depends_on: - redis-dev: - condition: service_healthy - db-dev: - condition: service_healthy - working_dir: /home/pn/app - entrypoint: /home/pn/dev-entrypoint.sh + shortlink: + <<: *main-app + command: ['docker/uwsgi/shortlink.ini'] + expose: + - 6410 ports: - - 3000:3000 - links: - - db-dev - - redis-dev - volumes: - # https://stackoverflow.com/questions/43844639/how-do-i-add-cached-or-delegated-into-a-docker-compose-yml-volumes-list - # https://forums.docker.com/t/what-happened-to-delegated-cached-ro-and-other-flags/105097/2 - - pip_cache:/home/pn/.cache/pip:delegated - - .:/home/pn/app - - node_modules:/home/pn/app/node_modules - - ./docker/entrypoints/dev.sh:/home/pn/dev-entrypoint.sh:ro - - ./instance/settings.py:/home/pn/app/instance/settings.py - environment: - - DB_HOST=db-dev - - POSTGRES_USER_HOST=funnel@db-dev - - REDIS_HOST=redis-dev - healthcheck: - test: bash -c '[[ "$$(curl -o /dev/null -s -w "%{http_code}\n" http://funnel.test:3000)" == "200" ]]' - interval: 30s - timeout: 1m - retries: 10 - start_period: 30s - asset-watcher: - extends: - file: docker/compose/services.yml - service: funnel - image: funnel-dev-asset-watcher - container_name: funnel-dev-asset-watcher - profiles: - - dev - build: - context: . - target: dev-assets - working_dir: /home/pn/app - entrypoint: npx webpack --mode development --watch - volumes: - - .:/home/pn/app - - node_modules:/home/pn/app/node_modules - environment: - - NODE_ENV=development - depends_on: - dev: - condition: service_healthy - healthcheck: - test: bash -c "[[ -f /home/pn/app/funnel/static/build/manifest.json ]]" - interval: 10s - timeout: 30s - retries: 60 - start_period: 1m - db-dev: - <<: *postgres - profiles: - - dev - - dev-no-watch - volumes: - - postgres_dev:/var/lib/postgresql/data - - ./docker/initdb/dev.sh:/docker-entrypoint-initdb.d/dev.sh:ro - healthcheck: - test: ['CMD-SHELL', 'psql funnel'] - redis-dev: - <<: *redis + - 6410:6410 + bash: + <<: *main-app + entrypoint: [''] profiles: - - dev - - dev-no-watch - volumes: - - redis_dev:/data + - maintenance redis: - <<: *redis - profiles: - - production + image: redis:latest + expose: + - 6379 + restart: always + healthcheck: + test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] volumes: - redis:/data -x-tmpfs: &tmpfs - driver: local - driver_opts: - type: tmpfs - device: tmpfs - o: 'uid=999,gid=999' # uid:gid is 999:999 for both postgres and redis volumes: - node_modules: - pip_cache: - postgres_dev: - redis_dev: redis: - postgres_test: - <<: *tmpfs - redis_test: - <<: *tmpfs diff --git a/docker/entrypoints/dev.sh b/docker/entrypoints/dev.sh deleted file mode 100755 index 9135c8ff1..000000000 --- a/docker/entrypoints/dev.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ "$(psql -XtA -U postgres -h $DB_HOST funnel -c "select count(*) from information_schema.tables where table_schema = 'public';")" = "0" ]; then - flask dbcreate - flask db stamp -fi - -./devserver.py diff --git a/docker/entrypoints/migrate.sh b/docker/entrypoints/migrate.sh new file mode 100755 index 000000000..d068de25a --- /dev/null +++ b/docker/entrypoints/migrate.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +if [ "$(psql -XtA -U postgres -h $DB_HOST $DB_FUNNEL -c "select count(*) from information_schema.tables where table_schema = 'public';")" = "0" ]; then + flask dbcreate + flask db stamp +else + flask db upgrade head +fi diff --git a/docker/uwsgi/funnel.ini b/docker/uwsgi/funnel.ini index 857590750..218eaa486 100644 --- a/docker/uwsgi/funnel.ini +++ b/docker/uwsgi/funnel.ini @@ -1,12 +1,12 @@ [uwsgi] -socket = 0.0.0.0:6400 +http = 0.0.0.0:6400 processes = 6 threads = 2 master = true uid = funnel gid = funnel -chdir = /home/pn/app +chdir = /home/funnel/app wsgi-file = wsgi.py callable = application buffer-size = 24000 -pidfile = /home/pn/%n.pid +pidfile = /home/funnel/%n.pid diff --git a/docker/uwsgi/shortlink.ini b/docker/uwsgi/shortlink.ini new file mode 100644 index 000000000..75ba8c985 --- /dev/null +++ b/docker/uwsgi/shortlink.ini @@ -0,0 +1,12 @@ +[uwsgi] +http = 0.0.0.0:6410 +processes = 2 +threads = 2 +master = true +uid = funnel +gid = funnel +chdir = /home/funnel/app +wsgi-file = wsgi.py +callable = shortlinkapp +buffer-size = 24000 +pidfile = /home/funnel/%n.pid diff --git a/sample.env b/sample.env index 105452ec3..593ae74c6 100644 --- a/sample.env +++ b/sample.env @@ -84,10 +84,11 @@ FLASK_CACHE_REDIS_URL=redis://${REDIS_HOST}:6379/0 # --- Database configuration DB_HOST=localhost +DB_FUNNEL=funnel # Main app database -FLASK_SQLALCHEMY_DATABASE_URI='postgresql+psycopg:///funnel' +FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg:///${DB_FUNNEL} # Geoname database (the use of `__` creates a dict and sets a key in the dict) -FLASK_SQLALCHEMY_BINDS__geoname='postgresql+psycopg:///geoname' +FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg:///geoname # --- Email configuration # SMTP mail server ('localhost' if Postfix is configured as a relay email server) From f48d90f5760744b74c0fa71f447fa5a459f46d66 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 25 Jun 2023 15:17:59 +0530 Subject: [PATCH 04/45] Add postfix setup to docker compose --- docker-compose.yml | 8 +++++++- docker/compose/services.yml | 20 -------------------- docker/entrypoints/migrate.sh | 8 -------- 3 files changed, 7 insertions(+), 29 deletions(-) delete mode 100644 docker/compose/services.yml delete mode 100755 docker/entrypoints/migrate.sh diff --git a/docker-compose.yml b/docker-compose.yml index 526e73e3b..0ad001d25 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,6 +45,12 @@ services: test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] volumes: - redis:/data - + postfix: + image: wildwildangel/postfix-relay:latest + env_file: + # https://github.com/sjinks/docker-alpine-postfix-relay + - .env.postfix + environment: + - SERVER_HOSTNAME=postfix volumes: redis: diff --git a/docker/compose/services.yml b/docker/compose/services.yml deleted file mode 100644 index 4bc90056e..000000000 --- a/docker/compose/services.yml +++ /dev/null @@ -1,20 +0,0 @@ -services: - funnel: - logging: - driver: json-file - options: - max-size: 200k - max-file: 10 - extra_hosts: - - 'funnel.test:127.0.0.1' - - 'f.test:127.0.0.1' - environment: - - FLASK_RUN_HOST=0.0.0.0 - funnel-prod: - extends: - file: services.yml - service: funnel - build: - target: production - links: - - redis diff --git a/docker/entrypoints/migrate.sh b/docker/entrypoints/migrate.sh deleted file mode 100755 index d068de25a..000000000 --- a/docker/entrypoints/migrate.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -if [ "$(psql -XtA -U postgres -h $DB_HOST $DB_FUNNEL -c "select count(*) from information_schema.tables where table_schema = 'public';")" = "0" ]; then - flask dbcreate - flask db stamp -else - flask db upgrade head -fi From 1003475fd91bc7f6dced996d701267e28c0e81a5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 25 Jun 2023 15:18:36 +0530 Subject: [PATCH 05/45] Update instructions in sample.env --- sample.env | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sample.env b/sample.env index 593ae74c6..084b4f72d 100644 --- a/sample.env +++ b/sample.env @@ -27,10 +27,12 @@ FLASK_DEBUG_TB_ENABLED=true # --- Domain configuration (these must point to 127.0.0.1 in /etc/hosts) # Funnel app's server name (Hasgeek uses 'hasgeek.com' in production) +# Use funnel.test:6400 for docker APP_FUNNEL_SERVER_NAME=funnel.test:3000 # Funnel app's default domain when running without a HTTP context APP_FUNNEL_DEFAULT_DOMAIN=funnel.test # Shortlink domain (Hasgeek uses 'has.gy' in production) +# Use f.test:6410 for docker FLASK_SHORTLINK_DOMAIN=f.test:3000 # Optional unsubscribe URL domain (Hasgeek uses 'bye.li' in production) # https://bye.li/* redirects to https://hasgeek.com/account/notifications/bye/* @@ -74,7 +76,7 @@ FLASK_SITE_SUPPORT_PHONE=+91... APP_FUNNEL_GA_CODE=null # --- Redis Queue and Redis cache (use separate dbs to isolate) -# Redis server host +# Redis server host. Use redis for docker. REDIS_HOST=localhost # RQ and cache FLASK_RQ_REDIS_URL=redis://${REDIS_HOST}:6379/1 @@ -83,6 +85,8 @@ FLASK_CACHE_TYPE=flask_caching.backends.RedisCache FLASK_CACHE_REDIS_URL=redis://${REDIS_HOST}:6379/0 # --- Database configuration +# Use host.docker.internal to connect to host system on docker +# Use postgres for test and dev setup DB_HOST=localhost DB_FUNNEL=funnel # Main app database @@ -92,6 +96,7 @@ FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg:///geoname # --- Email configuration # SMTP mail server ('localhost' if Postfix is configured as a relay email server) +# Use postfix for docker FLASK_MAIL_SERVER=localhost # If not using localhost, SMTP will need authentication # Port number (25 is default, but 587 is more likely for non-localhost) @@ -123,6 +128,7 @@ FLASK_TELEGRAM_ERROR_APIKEY=null # Send error reports to Slack webhooks (multiple); config spec TODO FLASK_SLACK_LOGGING_WEBHOOKS=null # Overrides for metrics being sent to telegraf +# Use host.docker.internal if running statsd on host machine on docker # STATSD_HOST=127.0.0.1 # STATSD_PORT=8125 # STATSD_IPV6=False From 7d78d7dbc1d9daedcfb997b8d9a36b627e62450a Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 25 Jun 2023 15:22:36 +0530 Subject: [PATCH 06/45] Include link to comprehensive dockerfile reference --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d270470e3..09197c7f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,7 @@ # syntax=docker/dockerfile:1.4 +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md + FROM node:lts-alpine as assets USER node WORKDIR /home/node/app From 27df0c5731ff1b7a09e2a4ec7b50bf32575c027f Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 25 Jun 2023 15:35:37 +0530 Subject: [PATCH 07/45] Configure postfix spool persistence in compose file --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 0ad001d25..1beee1278 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,5 +52,8 @@ services: - .env.postfix environment: - SERVER_HOSTNAME=postfix + volumes: + - postfix:/var/spool/postfix volumes: redis: + postfix: From e93f83d39319f3648df18e0c28c768085c636b5c Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Sun, 25 Jun 2023 19:26:19 +0530 Subject: [PATCH 08/45] Make postfix compose setup override-based --- docker-compose.postfix.yml | 20 ++++++++++++++++++++ docker-compose.yml | 10 ---------- 2 files changed, 20 insertions(+), 10 deletions(-) create mode 100644 docker-compose.postfix.yml diff --git a/docker-compose.postfix.yml b/docker-compose.postfix.yml new file mode 100644 index 000000000..4edc53398 --- /dev/null +++ b/docker-compose.postfix.yml @@ -0,0 +1,20 @@ +services: + main: + environment: + - FLASK_MAIL_SERVER=postfix + - FLASK_MAIL_PORT=25 + shortlink: + environment: + - FLASK_MAIL_SERVER=postfix + - FLASK_MAIL_PORT=25 + postfix: + image: wildwildangel/postfix-relay:latest + env_file: + # https://github.com/sjinks/docker-alpine-postfix-relay + - .env.postfix + environment: + - SERVER_HOSTNAME=postfix + volumes: + - postfix:/var/spool/postfix +volumes: + postfix: diff --git a/docker-compose.yml b/docker-compose.yml index 1beee1278..b4928e5ea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,15 +45,5 @@ services: test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] volumes: - redis:/data - postfix: - image: wildwildangel/postfix-relay:latest - env_file: - # https://github.com/sjinks/docker-alpine-postfix-relay - - .env.postfix - environment: - - SERVER_HOSTNAME=postfix - volumes: - - postfix:/var/spool/postfix volumes: redis: - postfix: From e2e29d799a306fad804fb5d93564d9ef41b1bd80 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 26 Jun 2023 08:02:18 +0530 Subject: [PATCH 09/45] CI pytest on docker --- ...{docker-ci-tests.yml => pytest-docker.yml} | 37 +++-------- Dockerfile | 22 +++++++ Makefile | 3 +- docker-compose.ci.yml | 62 +++++++++++++++++++ docker-compose.yml | 2 +- docker/.npmrc | 4 -- docker/entrypoints/ci-pre-test.sh | 6 -- docker/entrypoints/ci-test.sh | 5 -- docker/entrypoints/ci.sh | 6 ++ docker/images/bases.Dockerfile | 42 ------------- docker/initdb.sh | 20 ++++++ docker/initdb/dev.sh | 39 ------------ docker/initdb/test.sh | 20 ------ 13 files changed, 122 insertions(+), 146 deletions(-) rename .github/workflows/{docker-ci-tests.yml => pytest-docker.yml} (58%) create mode 100644 docker-compose.ci.yml delete mode 100644 docker/.npmrc delete mode 100755 docker/entrypoints/ci-pre-test.sh delete mode 100755 docker/entrypoints/ci-test.sh create mode 100755 docker/entrypoints/ci.sh delete mode 100644 docker/images/bases.Dockerfile create mode 100755 docker/initdb.sh delete mode 100755 docker/initdb/dev.sh delete mode 100755 docker/initdb/test.sh diff --git a/.github/workflows/docker-ci-tests.yml b/.github/workflows/pytest-docker.yml similarity index 58% rename from .github/workflows/docker-ci-tests.yml rename to .github/workflows/pytest-docker.yml index bf055aeeb..e26062d01 100644 --- a/.github/workflows/docker-ci-tests.yml +++ b/.github/workflows/pytest-docker.yml @@ -1,4 +1,4 @@ -name: 'Pytest on docker' +name: 'Pytest Docker' on: push: @@ -12,15 +12,17 @@ on: - '**.jinja2' - 'requirements/base.txt' - 'requirements/test.txt' - - '.github/workflows/docker-ci-tests.yml' + - '.github/workflows/pytest-docker.yml' - 'Dockerfile' + - 'Makefile' - 'pyproject.toml' - '.eslintrc.js' - 'docker-compose.yml' - - 'docker/compose/services.yml' - - 'docker/entrypoints/ci-test.sh' - - 'docker/initdb/test.sh' + - 'docker-compose.ci.yml' + - 'docker/entrypoints/ci.sh' + - 'docker/initdb.sh' - 'package.json' + - 'package-lock.json' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -30,26 +32,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - - name: Cache npm - uses: actions/cache@v3 - with: - path: .cache/.npm - key: docker-npm - - name: Cache node_modules - uses: actions/cache@v3 - with: - path: node_modules - key: docker-node_modules-${{ hashFiles('package-lock.json') }} - - name: Cache pip - uses: actions/cache@v3 - with: - path: .cache/pip - key: docker-pip - - name: Cache .local - uses: actions/cache@v3 - with: - path: .cache/.local - key: docker-user-local - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build funnel-test image @@ -57,8 +39,9 @@ jobs: uses: docker/build-push-action@v4 with: context: . - file: ci.Dockerfile - tags: funnel-test:latest + file: Dockerfile + target: ci + tags: funnel-ci load: true push: false - name: Run Tests diff --git a/Dockerfile b/Dockerfile index 09197c7f0..7a4115d8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,3 +42,25 @@ COPY --chown=funnel:funnel . . COPY --from=assets --chown=funnel:funnel /home/node/app/funnel/static/built/ funnel/static/build RUN mkdir -pv /home/funnel/app/logs ENTRYPOINT [ "uwsgi", "--ini" ] + +FROM app as ci +USER root +ENV PYTHONUNBUFFERED=1 +RUN mkdir -pv /home/funnel/app/coverage && chown -R funnel:funnel /home/funnel/.cache /home/funnel/app/coverage +# hadolint ignore=DL3008,DL4006,SC2046 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + apt-get update -yqq \ + && apt-get install -yqq --no-install-recommends xvfb firefox-esr \ + && apt-get autoclean -yqq \ + && apt-get autoremove -yqq \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest \ + | grep browser_download_url \ + | grep 'linux64.tar.gz\"' \ + | grep -o 'http.*\.gz') \ + | tar -xvz -C /usr/local/bin +USER funnel +COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test +RUN env +ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] diff --git a/Makefile b/Makefile index 4a7a1751d..85dc5805d 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,7 @@ docker-base-devtest: docker buildx build -f docker/images/bases.Dockerfile --target base-devtest --tag hasgeek/funnel-base-devtest . docker-ci-test: - COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain \ - docker compose --profile test up --quiet-pull --no-attach db-test --no-attach redis-test --no-log-prefix + docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --exit-code-from main docker-dev: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \ diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 000000000..093ef6788 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,62 @@ +name: funnel-ci +services: + main: + image: funnel-ci + build: + context: . + target: ci + links: + - db + - redis + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + restart: no + volumes: + - ./coverage:/home/funnel/app/coverage + environment: + - REDIS_HOST=redis + - FLASK_RUN_HOST=host.docker.internal + - DB_HOST=host.docker.internal + - DB_FUNNEL=funnel + - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel + - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname + redis: + image: redis:latest + expose: + - 6379 + restart: always + healthcheck: + test: ['CMD', 'redis-cli', '--raw', 'incr', 'ping'] + volumes: + - redis:/data + db: + image: postgres:latest + restart: always + user: postgres + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + - POSTGRES_USER=postgres + healthcheck: + interval: 5s + timeout: 5s + retries: 5 + test: ['CMD-SHELL', 'psql funnel'] + volumes: + - db:/var/lib/postgresql/data + - ./docker/initdb.sh:/docker-entrypoint-initdb.d/initdb.sh:ro +volumes: + redis: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: 'uid=999,gid=999' # uid:gid is 999:999 for redis + db: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: 'uid=999,gid=999' # uid:gid is 999:999 for redis diff --git a/docker-compose.yml b/docker-compose.yml index b4928e5ea..79437fdb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: depends_on: - redis env_file: - - .env + - .env.production environment: - REDIS_HOST=redis - FLASK_RUN_HOST=host.docker.internal diff --git a/docker/.npmrc b/docker/.npmrc deleted file mode 100644 index 7a650526c..000000000 --- a/docker/.npmrc +++ /dev/null @@ -1,4 +0,0 @@ -audit = false -fund = false -loglevel = warn -update-notifier = false diff --git a/docker/entrypoints/ci-pre-test.sh b/docker/entrypoints/ci-pre-test.sh deleted file mode 100755 index 29a558f0a..000000000 --- a/docker/entrypoints/ci-pre-test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# https://github.com/docker/for-mac/issues/5480 - -chown -R pn:pn /home/pn/.npm /home/pn/.cache /home/pn/.cache/pip /home/pn/app \ - /home/pn/app/coverage /home/pn/.local diff --git a/docker/entrypoints/ci-test.sh b/docker/entrypoints/ci-test.sh deleted file mode 100755 index 4da374e54..000000000 --- a/docker/entrypoints/ci-test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -make install-test -pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts db-test | awk '/STREAM/ { print $1}'),$(getent ahosts redis-test | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel -coverage lcov -o coverage/funnel.lcov diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh new file mode 100755 index 000000000..c796312c9 --- /dev/null +++ b/docker/entrypoints/ci.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +pytest \ + --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ + --gherkin-terminal-reporter -vv --showlocals --cov=funnel $@ +coverage lcov -o coverage/funnel.lcov diff --git a/docker/images/bases.Dockerfile b/docker/images/bases.Dockerfile deleted file mode 100644 index 9e27b0cf7..000000000 --- a/docker/images/bases.Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# syntax=docker/dockerfile:1.4 - -# Dockerfile syntax & features documentation: -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md - -FROM nikolaik/python-nodejs:python3.11-nodejs20-bullseye as base - -# https://github.com/zalando/postgres-operator/blob/master/docker/logical-backup/Dockerfile -# https://stackoverflow.com/questions/68465355/what-is-the-meaning-of-set-o-pipefail-in-bash-script -SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] - -STOPSIGNAL SIGINT -ENV PATH "$PATH:/home/pn/.local/bin" - -# Install postgresql-client-15 -USER root:root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -y \ - && apt-get install -y --no-install-recommends lsb-release \ - && echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ - && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ - && apt-get update -y && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends postgresql-client-15 \ - && apt-get purge -y lsb-release -RUN mkdir -pv /var/cache/funnel && chown -R pn:pn /var/cache/funnel -USER pn:pn - -FROM base as base-devtest -# Install firefox & geckodriver -USER root:root -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt-get update -y \ - && apt-get upgrade -y \ - && echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firefox-esr \ - && cd /tmp/ \ - && curl -fsSL $(curl -fsSL https://api.github.com/repos/mozilla/geckodriver/releases/latest | grep browser_download_url | grep 'linux64.tar.gz\"'| grep -o 'http.*\.gz') > gecko.tar.gz \ - && tar -xvzf gecko.tar.gz \ - && rm gecko.tar.gz \ - && chmod +x geckodriver \ - && mv geckodriver /usr/local/bin -USER pn:pn diff --git a/docker/initdb.sh b/docker/initdb.sh new file mode 100755 index 000000000..cb87cbb0b --- /dev/null +++ b/docker/initdb.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +psql -c "create user funnel;" +psql -c "create database funnel;" +psql -c "create database geoname;" +psql funnel << $$ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS unaccent; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +$$ +psql geoname << $$ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS unaccent; +CREATE EXTENSION IF NOT EXISTS pgcrypto; +$$ +psql -c "grant all privileges on database funnel to funnel;" +psql -c "grant all privileges on database geoname to funnel;" +psql funnel -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" +psql geoname -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" diff --git a/docker/initdb/dev.sh b/docker/initdb/dev.sh deleted file mode 100755 index 4cc3bf36d..000000000 --- a/docker/initdb/dev.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -e - -psql -c "create user funnel;" -psql -c "create database funnel;" -psql -c "create database geoname;" -psql -c "create database funnel_testing;" -psql -c "create database geoname_testing;" -psql funnel << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ - -psql funnel_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ - -psql -c "grant all privileges on database funnel to funnel;" -psql -c "grant all privileges on database geoname to funnel;" -psql funnel -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" - -psql -c "grant all privileges on database funnel_testing to funnel;" -psql -c "grant all privileges on database geoname_testing to funnel;" -psql funnel_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" diff --git a/docker/initdb/test.sh b/docker/initdb/test.sh deleted file mode 100755 index 2d7e0962f..000000000 --- a/docker/initdb/test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e - -psql -c "create user funnel;" -psql -c "create database funnel_testing;" -psql -c "create database geoname_testing;" -psql funnel_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql geoname_testing << $$ -CREATE EXTENSION IF NOT EXISTS pg_trgm; -CREATE EXTENSION IF NOT EXISTS unaccent; -CREATE EXTENSION IF NOT EXISTS pgcrypto; -$$ -psql -c "grant all privileges on database funnel_testing to funnel;" -psql -c "grant all privileges on database geoname_testing to funnel;" -psql funnel_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" -psql geoname_testing -c "grant all privileges on schema public to funnel; grant all privileges on all tables in schema public to funnel; grant all privileges on all sequences in schema public to funnel;" From 3aef27040003e6faa0e364a36bfbe82a18d15189 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 26 Jun 2023 10:57:11 +0530 Subject: [PATCH 10/45] Remove a debug instruction from Dockerfile --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7a4115d8c..b85a0df1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,5 +62,4 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ USER funnel COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test -RUN env ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] From 38637792c0edbfe6b29611bc65b0f5342b6ac650 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 26 Jun 2023 11:25:36 +0530 Subject: [PATCH 11/45] Add extra_hosts to CI compose file --- docker-compose.ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 093ef6788..7feb45466 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -23,6 +23,10 @@ services: - DB_FUNNEL=funnel - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname + extra_hosts: + - funnel.test:127.0.0.1 + - f.test:127.0.0.1 + - bye.test:127.0.0.1 redis: image: redis:latest expose: From 2e05d9eb13d348a60366cd4baf6033544bc28e31 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Mon, 26 Jun 2023 12:16:25 +0530 Subject: [PATCH 12/45] Create coverage directory as funnel instead of root in Dockerfile ci stage --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b85a0df1b..34a55a82d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ENTRYPOINT [ "uwsgi", "--ini" ] FROM app as ci USER root ENV PYTHONUNBUFFERED=1 -RUN mkdir -pv /home/funnel/app/coverage && chown -R funnel:funnel /home/funnel/.cache /home/funnel/app/coverage +RUN chown -R funnel:funnel /home/funnel/.cache # hadolint ignore=DL3008,DL4006,SC2046 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update -yqq \ @@ -60,6 +60,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ | grep -o 'http.*\.gz') \ | tar -xvz -C /usr/local/bin USER funnel +RUN mkdir -pv /home/funnel/app/coverage COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] From 65a40b30292e12351fcd116d1c89492a8c531541 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 01:50:05 +0530 Subject: [PATCH 13/45] Try to create a coverage folder in GHA host and set it's ownership before mounting it --- .github/workflows/pytest-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index e26062d01..51aeb3693 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -34,6 +34,8 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Set permissions for coverage folder + run: mkdir -pv coverage && chown -R 1000:1000 coverage - name: Build funnel-test image id: build-funnel-test uses: docker/build-push-action@v4 From 1f4c3d217782a236c1447690cf05a394186e8ed3 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 02:14:39 +0530 Subject: [PATCH 14/45] Create & set permissions of coverage folder with uid and gid in Dockerfile --- .github/workflows/pytest-docker.yml | 2 -- Dockerfile | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 51aeb3693..e26062d01 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -34,8 +34,6 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Set permissions for coverage folder - run: mkdir -pv coverage && chown -R 1000:1000 coverage - name: Build funnel-test image id: build-funnel-test uses: docker/build-push-action@v4 diff --git a/Dockerfile b/Dockerfile index 34a55a82d..a76aacf8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ENTRYPOINT [ "uwsgi", "--ini" ] FROM app as ci USER root ENV PYTHONUNBUFFERED=1 -RUN chown -R funnel:funnel /home/funnel/.cache +RUN mkdir -pv /home/funnel/app/coverage && chown -R 1000:1000 /home/funnel/.cache /home/funnel/app/coverage # hadolint ignore=DL3008,DL4006,SC2046 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update -yqq \ @@ -60,7 +60,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ | grep -o 'http.*\.gz') \ | tar -xvz -C /usr/local/bin USER funnel -RUN mkdir -pv /home/funnel/app/coverage COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] From 0d2a4915ad0cc5fdef63de27e41543a9905a113f Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 02:43:07 +0530 Subject: [PATCH 15/45] Alternate approach to use docker compose run to pre-set permission of coverage folder --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 85dc5805d..21fa3118a 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ docker-base-devtest: docker buildx build -f docker/images/bases.Dockerfile --target base-devtest --tag hasgeek/funnel-base-devtest . docker-ci-test: + docker compose -f docker-compose.ci.yml run -u root --entrypoint "" main chown -R 1000:1000 coverage && \ docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --exit-code-from main docker-dev: From 13f3d485ca23c1bb0381634badca64a94ba1c730 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 03:03:47 +0530 Subject: [PATCH 16/45] =?UTF-8?q?Make=20CI=20tests=20fail=20to=20check=20i?= =?UTF-8?q?f=20--exit-code-from=20works=20to=20mark=20GHA=20step=20for=20p?= =?UTF-8?q?ytest=20to=20also=20fail=CB=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 7feb45466..a733f45a0 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -20,7 +20,7 @@ services: - REDIS_HOST=redis - FLASK_RUN_HOST=host.docker.internal - DB_HOST=host.docker.internal - - DB_FUNNEL=funnel + - DB_FUNNEL=funnels - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname extra_hosts: From 7a744678f2dde8d5370985cb416939bdbfd4cefb Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 03:15:35 +0530 Subject: [PATCH 17/45] =?UTF-8?q?Make=20CI=20tests=20fail=20to=20check=20i?= =?UTF-8?q?f=20--exit-code-from=20works=20to=20mark=20GHA=20step=20for=20p?= =?UTF-8?q?ytest=20to=20also=20fail=CB=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index a733f45a0..9a8320905 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -19,8 +19,8 @@ services: environment: - REDIS_HOST=redis - FLASK_RUN_HOST=host.docker.internal - - DB_HOST=host.docker.internal - - DB_FUNNEL=funnels + - DB_HOST=localhost + - DB_FUNNEL=funnel - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname extra_hosts: From 76ec7931328c30a4fd89ca588c23d162d07c5793 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 03:34:38 +0530 Subject: [PATCH 18/45] =?UTF-8?q?Make=20CI=20tests=20fail=20to=20check=20i?= =?UTF-8?q?f=20--exit-code-from=20works=20to=20mark=20GHA=20step=20for=20p?= =?UTF-8?q?ytest=20to=20also=20fail=CB=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.ci.yml | 2 +- tests/unit/forms/account_test.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9a8320905..7feb45466 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -19,7 +19,7 @@ services: environment: - REDIS_HOST=redis - FLASK_RUN_HOST=host.docker.internal - - DB_HOST=localhost + - DB_HOST=host.docker.internal - DB_FUNNEL=funnel - FLASK_SQLALCHEMY_DATABASE_URI=postgresql+psycopg://funnel@db/funnel - FLASK_SQLALCHEMY_BINDS__geoname=postgresql+psycopg://funnel@db/geoname diff --git a/tests/unit/forms/account_test.py b/tests/unit/forms/account_test.py index 99bdabcc8..a8487ce05 100644 --- a/tests/unit/forms/account_test.py +++ b/tests/unit/forms/account_test.py @@ -37,6 +37,7 @@ def form(request): @pytest.mark.formdata() def test_password_policy_form_no_data(form) -> None: """Test form validation for missing password.""" + pytest.fail('Deliberate failure for testing CI') assert form.validate() is False From 9fe1352e5bf9e29f4530a47b09c501912403ebb5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 05:08:18 +0530 Subject: [PATCH 19/45] Use pytest-github-actions-annotate-failures in the CI docker image --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a76aacf8a..a6d80d73f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ ENTRYPOINT [ "uwsgi", "--ini" ] FROM app as ci USER root -ENV PYTHONUNBUFFERED=1 RUN mkdir -pv /home/funnel/app/coverage && chown -R 1000:1000 /home/funnel/.cache /home/funnel/app/coverage # hadolint ignore=DL3008,DL4006,SC2046 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -60,6 +59,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ | grep -o 'http.*\.gz') \ | tar -xvz -C /usr/local/bin USER funnel +ENV PYTHONUNBUFFERED=1 +ENV GITHUB_ACTIONS=true +# hadolint ignore=DL3013,DL3042 +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 pip install pytest-github-actions-annotate-failures COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] From aef75daf7b654256125e2cb484c94512b555631e Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 05:41:19 +0530 Subject: [PATCH 20/45] Try running tests with docker run in the GHA pytest docker workflow --- .github/workflows/pytest-docker.yml | 8 ++++++-- Dockerfile | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index e26062d01..59a95a724 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -44,8 +44,12 @@ jobs: tags: funnel-ci load: true push: false - - name: Run Tests - run: make docker-ci-test + - name: Set permissions for coverage + run: docker compose -f docker-compose.ci.yml run -u root --quiet-pull main chown -R 1000:1000 coverage + - name: Test with pytest + run: docker compose -f docker-compose.ci.yml run main pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel + - name: Prepare coverage report + run: docker compose -f docker-compose.ci.yml run main coverage lcov -o coverage/funnel.lcov - name: Upload coverage report to Coveralls uses: coverallsapp/github-action@master with: diff --git a/Dockerfile b/Dockerfile index a6d80d73f..13b8fe783 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,4 +65,4 @@ ENV GITHUB_ACTIONS=true RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 pip install pytest-github-actions-annotate-failures COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test -ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] +ENTRYPOINT [ "" ] From dfd892f82fe557e9eab21e3ef25fb16365905f9e Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 19:35:00 +0530 Subject: [PATCH 21/45] Try running tests with docker run in the GHA pytest docker workflow --- .github/workflows/pytest-docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 59a95a724..9675ea944 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -45,11 +45,11 @@ jobs: load: true push: false - name: Set permissions for coverage - run: docker compose -f docker-compose.ci.yml run -u root --quiet-pull main chown -R 1000:1000 coverage + run: docker compose -f docker-compose.ci.yml run -u root --quiet-pull main "chown -R 1000:1000 coverage" - name: Test with pytest - run: docker compose -f docker-compose.ci.yml run main pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel + run: docker compose -f docker-compose.ci.yml run main "pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel" - name: Prepare coverage report - run: docker compose -f docker-compose.ci.yml run main coverage lcov -o coverage/funnel.lcov + run: docker compose -f docker-compose.ci.yml run main "coverage lcov -o coverage/funnel.lcov" - name: Upload coverage report to Coveralls uses: coverallsapp/github-action@master with: From 08b7655493c99a0aee35eba7d35fad5faa25e874 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 19:55:34 +0530 Subject: [PATCH 22/45] Try running tests with docker run in the GHA pytest docker workflow --- .github/workflows/pytest-docker.yml | 6 +++--- Dockerfile | 2 +- docker/entrypoints/ci.sh | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 9675ea944..2909aeed6 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -45,11 +45,11 @@ jobs: load: true push: false - name: Set permissions for coverage - run: docker compose -f docker-compose.ci.yml run -u root --quiet-pull main "chown -R 1000:1000 coverage" + run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest - run: docker compose -f docker-compose.ci.yml run main "pytest --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') --gherkin-terminal-reporter -vv --showlocals --cov=funnel" + run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis - name: Prepare coverage report - run: docker compose -f docker-compose.ci.yml run main "coverage lcov -o coverage/funnel.lcov" + run: docker compose -f docker-compose.ci.yml run --entrypoint "" main "coverage lcov -o coverage/funnel.lcov" - name: Upload coverage report to Coveralls uses: coverallsapp/github-action@master with: diff --git a/Dockerfile b/Dockerfile index 13b8fe783..a6d80d73f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,4 +65,4 @@ ENV GITHUB_ACTIONS=true RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 pip install pytest-github-actions-annotate-failures COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test -ENTRYPOINT [ "" ] +ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh index c796312c9..fc43924da 100755 --- a/docker/entrypoints/ci.sh +++ b/docker/entrypoints/ci.sh @@ -3,4 +3,3 @@ pytest \ --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ --gherkin-terminal-reporter -vv --showlocals --cov=funnel $@ -coverage lcov -o coverage/funnel.lcov From c96f714f19a006e3222fd137b442af7efc03db26 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 20:13:07 +0530 Subject: [PATCH 23/45] Try running tests with docker run in the GHA pytest docker workflow --- .github/workflows/pytest-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 2909aeed6..244c8d757 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -47,9 +47,9 @@ jobs: - name: Set permissions for coverage run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest - run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis + run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --exit-code-from main - name: Prepare coverage report - run: docker compose -f docker-compose.ci.yml run --entrypoint "" main "coverage lcov -o coverage/funnel.lcov" + run: docker compose -f docker-compose.ci.yml run --entrypoint "" main coverage lcov -o coverage/funnel.lcov - name: Upload coverage report to Coveralls uses: coverallsapp/github-action@master with: From f814079315ce44ea62a018d6e64b6c8fbed20f7e Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 20:38:51 +0530 Subject: [PATCH 24/45] Try running tests with docker run in the GHA pytest docker workflow --- docker/entrypoints/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh index fc43924da..f64921cb2 100755 --- a/docker/entrypoints/ci.sh +++ b/docker/entrypoints/ci.sh @@ -1,5 +1,5 @@ #!/bin/sh -pytest \ +GITHUB_ACTIONS=true pytest \ --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ --gherkin-terminal-reporter -vv --showlocals --cov=funnel $@ From e4fad2ed313caeac8ad5c6465ea6ad9ccb4da76b Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 21:06:09 +0530 Subject: [PATCH 25/45] Added --no-log-prefix to the pytest step - log prefixes maybe interfering with the pytest annotations plugin's output, which is supposed to begin with ::error --- .github/workflows/pytest-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 244c8d757..f04297455 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -45,9 +45,9 @@ jobs: load: true push: false - name: Set permissions for coverage - run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --quiet-pull main chown -R 1000:1000 coverage + run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest - run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --exit-code-from main + run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main - name: Prepare coverage report run: docker compose -f docker-compose.ci.yml run --entrypoint "" main coverage lcov -o coverage/funnel.lcov - name: Upload coverage report to Coveralls From 04a792203449c19860fe27cb42bfe1250eece5a1 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 21:23:03 +0530 Subject: [PATCH 26/45] Remove deliberate test failure as pytest failure annotations now work with GHA workflow for docker based pytest --- tests/unit/forms/account_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/forms/account_test.py b/tests/unit/forms/account_test.py index a8487ce05..99bdabcc8 100644 --- a/tests/unit/forms/account_test.py +++ b/tests/unit/forms/account_test.py @@ -37,7 +37,6 @@ def form(request): @pytest.mark.formdata() def test_password_policy_form_no_data(form) -> None: """Test form validation for missing password.""" - pytest.fail('Deliberate failure for testing CI') assert form.validate() is False From a5575d40157f2819792333e2bca11f11fc499802 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 27 Jun 2023 21:39:09 +0530 Subject: [PATCH 27/45] Move coverage report preparation for docker-based CI back into entrypoint shell script --- .github/workflows/pytest-docker.yml | 2 -- docker/entrypoints/ci.sh | 3 ++- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index f04297455..13c362ebe 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -48,8 +48,6 @@ jobs: run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main - - name: Prepare coverage report - run: docker compose -f docker-compose.ci.yml run --entrypoint "" main coverage lcov -o coverage/funnel.lcov - name: Upload coverage report to Coveralls uses: coverallsapp/github-action@master with: diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh index f64921cb2..c796312c9 100755 --- a/docker/entrypoints/ci.sh +++ b/docker/entrypoints/ci.sh @@ -1,5 +1,6 @@ #!/bin/sh -GITHUB_ACTIONS=true pytest \ +pytest \ --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ --gherkin-terminal-reporter -vv --showlocals --cov=funnel $@ +coverage lcov -o coverage/funnel.lcov From a8a6be1e37f087926ddffce5d27b142979f2bb9a Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 04:55:16 +0530 Subject: [PATCH 28/45] Add pytest-github-actions-annotate-failures to python dependencies for test --- Dockerfile | 2 -- requirements/base.txt | 13 +------------ requirements/dev.txt | 2 ++ requirements/test.in | 1 + requirements/test.txt | 11 ++++++----- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index a6d80d73f..9715014e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,8 +61,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ USER funnel ENV PYTHONUNBUFFERED=1 ENV GITHUB_ACTIONS=true -# hadolint ignore=DL3013,DL3042 -RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 pip install pytest-github-actions-annotate-failures COPY --chown=funnel:funnel requirements/base.txt requirements/test.txt ./requirements/ RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python-test ENTRYPOINT [ "/home/funnel/app/docker/entrypoints/ci.sh" ] diff --git a/requirements/base.txt b/requirements/base.txt index a18ecd936..7259a832a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -38,9 +38,7 @@ arrow==1.2.3 asgiref==3.7.2 # via -r requirements/base.in async-timeout==4.0.2 - # via - # aiohttp - # redis + # via aiohttp attrs==23.1.0 # via aiohttp babel==2.12.1 @@ -117,8 +115,6 @@ dnspython==2.3.0 # pyisemail emoji==2.5.0 # via baseframe -exceptiongroup==1.1.1 - # via anyio filelock==3.12.1 # via tldextract flask==2.2.5 @@ -225,10 +221,6 @@ idna==3.4 # requests # tldextract # yarl -importlib-metadata==6.6.0 - # via - # flask - # markdown isoweek==1.3.3 # via coaster itsdangerous==2.1.2 @@ -491,7 +483,6 @@ typing-extensions==4.6.3 # via # -r requirements/base.in # alembic - # asgiref # baseframe # coaster # psycopg @@ -539,8 +530,6 @@ wtforms-sqlalchemy==0.3 # via baseframe yarl==1.9.2 # via aiohttp -zipp==3.15.0 - # via importlib-metadata zxcvbn==4.4.28 # via -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 7eb93df98..86a4cf912 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -141,6 +141,8 @@ stevedore==5.1.0 # via bandit tokenize-rt==5.1.0 # via pyupgrade +tomli==2.0.1 + # via -r requirements/dev.in toposort==1.10 # via pip-compile-multi types-geoip2==3.0.0 diff --git a/requirements/test.in b/requirements/test.in index 3d5612615..5dedb12b7 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -11,6 +11,7 @@ pytest-bdd pytest-cov pytest-dotenv pytest-env +pytest-github-actions-annotate-failures pytest-rerunfailures pytest-selenium>=4.0.1 pytest-socket diff --git a/requirements/test.txt b/requirements/test.txt index 70e3649ff..0b3bda5d0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ -# SHA1:4acb164f314e8087c9b429568f0005b6e65c5b88 +# SHA1:9e8b9262668d4c9228abc826cefaf77fde9eeddf # # This file is autogenerated by pip-compile-multi # To update, run: @@ -27,6 +27,8 @@ coveralls==3.3.1 # via -r requirements/test.in docopt==0.6.2 # via coveralls +exceptiongroup==1.1.1 + # via trio-websocket iniconfig==2.0.0 # via pytest outcome==1.2.0 @@ -50,6 +52,7 @@ pytest==7.3.2 # pytest-cov # pytest-dotenv # pytest-env + # pytest-github-actions-annotate-failures # pytest-html # pytest-metadata # pytest-rerunfailures @@ -68,6 +71,8 @@ pytest-dotenv==0.5.2 # via -r requirements/test.in pytest-env==0.8.1 # via -r requirements/test.in +pytest-github-actions-annotate-failures==0.2.0 + # via -r requirements/test.in pytest-html==3.2.0 # via pytest-selenium pytest-metadata==3.0.0 @@ -94,10 +99,6 @@ sttable==0.0.1 # via -r requirements/test.in tenacity==8.2.2 # via pytest-selenium -tomli==2.0.1 - # via - # coverage - # pytest tomlkit==0.11.8 # via -r requirements/test.in trio==0.22.0 From f95e7bf121fab0e498803edb8ba1a016e973091b Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 05:24:53 +0530 Subject: [PATCH 29/45] Update trigger conditions for pytest docker GHA workflow --- .github/workflows/pytest-docker.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 13c362ebe..4fbe9b440 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -2,22 +2,19 @@ name: 'Pytest Docker' on: push: - branches: ['main'] - pull_request: - branches: ['main'] paths: - '**.py' - '**.js' - '**.scss' - '**.jinja2' + - '.flaskenv' + - '.testenv' - 'requirements/base.txt' - 'requirements/test.txt' - '.github/workflows/pytest-docker.yml' - 'Dockerfile' - 'Makefile' - 'pyproject.toml' - - '.eslintrc.js' - - 'docker-compose.yml' - 'docker-compose.ci.yml' - 'docker/entrypoints/ci.sh' - 'docker/initdb.sh' From 9e381f84f5b5578242821ce9ecdf7b5c5bdf35f5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 05:37:44 +0530 Subject: [PATCH 30/45] Add the gha build cache backend for the funnel-ci build step in the pytest docker workflow --- .github/workflows/pytest-docker.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 4fbe9b440..8fcb96c2e 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -31,8 +31,8 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build funnel-test image - id: build-funnel-test + - name: Build funnel-ci image + id: build-funnel-ci uses: docker/build-push-action@v4 with: context: . @@ -41,6 +41,8 @@ jobs: tags: funnel-ci load: true push: false + cache-from: type=gha + cache-to: type=gha,mode=max - name: Set permissions for coverage run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest From e6408f1743eab42a387e95ffbf709ed65662fe03 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 17:43:19 +0530 Subject: [PATCH 31/45] Add step to build funnel production image in the pytest docker workflow to check if it already uses the local build cache from previous ci build in same run --- .github/workflows/pytest-docker.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 8fcb96c2e..06fcea211 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -53,3 +53,12 @@ jobs: github-token: ${{ secrets.github_token }} path-to-lcov: coverage/funnel.lcov flag-name: docker-3.11 + - name: Build funnel image + id: build-funnel + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + target: app + tags: funnel + push: false From 2481c6546051fb43a06e6e0c027355071fa36b98 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 20:34:28 +0530 Subject: [PATCH 32/45] Update python dependencies --- requirements/base.txt | 22 +++++++++++----------- requirements/dev.txt | 12 ++++++------ requirements/test.txt | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 586ad9ec9..5f972bdd0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -62,9 +62,9 @@ blinker==1.6.2 # -r requirements/base.in # baseframe # coaster -boto3==1.26.156 +boto3==1.26.162 # via -r requirements/base.in -botocore==1.29.156 +botocore==1.29.162 # via # boto3 # s3transfer @@ -113,7 +113,7 @@ dnspython==2.3.0 # baseframe # mxsniff # pyisemail -emoji==2.5.1 +emoji==2.6.0 # via baseframe filelock==3.12.2 # via tldextract @@ -159,7 +159,7 @@ flask-redis==0.4.0 # via -r requirements/base.in flask-rq2==18.3 # via -r requirements/base.in -flask-sqlalchemy==3.0.4 +flask-sqlalchemy==3.0.5 # via # -r requirements/base.in # coaster @@ -237,7 +237,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.2.0 +joblib==1.3.0 # via nltk linkify-it-py==2.0.2 # via -r requirements/base.in @@ -301,7 +301,7 @@ packaging==23.1 # via marshmallow passlib==1.7.4 # via -r requirements/base.in -phonenumbers==8.13.14 +phonenumbers==8.13.15 # via -r requirements/base.in premailer==3.10.0 # via -r requirements/base.in @@ -374,7 +374,7 @@ pyyaml==6.0 # pymdown-extensions qrcode==7.4.2 # via -r requirements/base.in -redis==4.5.5 +redis==4.6.0 # via # baseframe # flask-redis @@ -407,7 +407,7 @@ requests-oauthlib==1.3.1 # via tweepy rich==13.4.2 # via -r requirements/base.in -rq==1.15.0 +rq==1.15.1 # via # -r requirements/base.in # baseframe @@ -426,7 +426,7 @@ semantic-version==2.10.0 # via # baseframe # coaster -sentry-sdk==1.25.1 +sentry-sdk==1.26.0 # via baseframe six==1.16.0 # via @@ -446,7 +446,7 @@ sniffio==1.3.0 # anyio # httpcore # httpx -sqlalchemy==2.0.16 +sqlalchemy==2.0.17 # via # -r requirements/base.in # alembic @@ -477,7 +477,7 @@ tuspy==1.0.1 # via pyvimeo tweepy==4.14.0 # via -r requirements/base.in -twilio==8.3.0 +twilio==8.4.0 # via -r requirements/base.in typing-extensions==4.6.3 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index 0226b21b5..ff2f24bac 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -61,7 +61,7 @@ flake8-logging-format==0.9.0 # via -r requirements/dev.in flake8-mutable==1.2.0 # via -r requirements/dev.in -flake8-plugin-utils==1.3.2 +flake8-plugin-utils==1.3.3 # via flake8-pytest-style flake8-print==5.0.0 # via -r requirements/dev.in @@ -90,7 +90,7 @@ mccabe==0.7.0 # via # flake8 # pylint -mypy==1.4.0 +mypy==1.4.1 # via -r requirements/dev.in mypy-json-report==1.0.4 # via -r requirements/dev.in @@ -106,7 +106,7 @@ pip-compile-multi==2.6.3 # via -r requirements/dev.in pip-tools==6.13.0 # via pip-compile-multi -platformdirs==3.6.0 +platformdirs==3.8.0 # via # black # pylint @@ -131,7 +131,7 @@ pyupgrade==3.7.0 # via -r requirements/dev.in reformat-gherkin==3.0.1 # via -r requirements/dev.in -ruff==0.0.272 +ruff==0.0.275 # via -r requirements/dev.in smmap==5.0.0 # via gitdb @@ -151,13 +151,13 @@ types-ipaddress==1.0.8 # via types-maxminddb types-maxminddb==1.5.0 # via types-geoip2 -types-pyopenssl==23.2.0.0 +types-pyopenssl==23.2.0.1 # via types-redis types-python-dateutil==2.8.19.13 # via -r requirements/dev.in types-pytz==2023.3.0.0 # via -r requirements/dev.in -types-redis==4.5.5.2 +types-redis==4.6.0.0 # via -r requirements/dev.in types-requests==2.31.0.1 # via -r requirements/dev.in diff --git a/requirements/test.txt b/requirements/test.txt index c0dbfe9ba..5753ae509 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -39,11 +39,11 @@ parse==1.19.1 # pytest-bdd parse-type==0.6.0 # via pytest-bdd -pluggy==1.0.0 +pluggy==1.2.0 # via pytest py==1.11.0 # via pytest-html -pytest==7.3.2 +pytest==7.4.0 # via # -r requirements/test.in # pytest-asyncio @@ -89,7 +89,7 @@ requests-mock==1.11.0 # via -r requirements/test.in respx==0.20.1 # via -r requirements/test.in -selenium==4.9.1 +selenium==4.10.0 # via pytest-selenium sortedcontainers==2.4.0 # via trio From 554434ea92bf14d1364ac1316acd07dfbd9d8a92 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 22:39:51 +0530 Subject: [PATCH 33/45] Change entrypoint shell script for docker CI to not exit with code 0 for pytest failures --- docker/entrypoints/ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/entrypoints/ci.sh b/docker/entrypoints/ci.sh index c796312c9..23386f5eb 100755 --- a/docker/entrypoints/ci.sh +++ b/docker/entrypoints/ci.sh @@ -1,4 +1,6 @@ -#!/bin/sh +#!/bin/bash + +set -e pytest \ --allow-hosts=127.0.0.1,::1,$(hostname -i),$(getent ahosts $DB_HOST | awk '/STREAM/ { print $1}'),$(getent ahosts $REDIS_HOST | awk '/STREAM/ { print $1}') \ From 333dff3afd16cf4fa2ff4edae1dc5302582e070b Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 28 Jun 2023 22:50:56 +0530 Subject: [PATCH 34/45] Revert "Update python dependencies" This reverts commit 2481c6546051fb43a06e6e0c027355071fa36b98. --- requirements/base.txt | 22 +++++++++++----------- requirements/dev.txt | 12 ++++++------ requirements/test.txt | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5f972bdd0..586ad9ec9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -62,9 +62,9 @@ blinker==1.6.2 # -r requirements/base.in # baseframe # coaster -boto3==1.26.162 +boto3==1.26.156 # via -r requirements/base.in -botocore==1.29.162 +botocore==1.29.156 # via # boto3 # s3transfer @@ -113,7 +113,7 @@ dnspython==2.3.0 # baseframe # mxsniff # pyisemail -emoji==2.6.0 +emoji==2.5.1 # via baseframe filelock==3.12.2 # via tldextract @@ -159,7 +159,7 @@ flask-redis==0.4.0 # via -r requirements/base.in flask-rq2==18.3 # via -r requirements/base.in -flask-sqlalchemy==3.0.5 +flask-sqlalchemy==3.0.4 # via # -r requirements/base.in # coaster @@ -237,7 +237,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.3.0 +joblib==1.2.0 # via nltk linkify-it-py==2.0.2 # via -r requirements/base.in @@ -301,7 +301,7 @@ packaging==23.1 # via marshmallow passlib==1.7.4 # via -r requirements/base.in -phonenumbers==8.13.15 +phonenumbers==8.13.14 # via -r requirements/base.in premailer==3.10.0 # via -r requirements/base.in @@ -374,7 +374,7 @@ pyyaml==6.0 # pymdown-extensions qrcode==7.4.2 # via -r requirements/base.in -redis==4.6.0 +redis==4.5.5 # via # baseframe # flask-redis @@ -407,7 +407,7 @@ requests-oauthlib==1.3.1 # via tweepy rich==13.4.2 # via -r requirements/base.in -rq==1.15.1 +rq==1.15.0 # via # -r requirements/base.in # baseframe @@ -426,7 +426,7 @@ semantic-version==2.10.0 # via # baseframe # coaster -sentry-sdk==1.26.0 +sentry-sdk==1.25.1 # via baseframe six==1.16.0 # via @@ -446,7 +446,7 @@ sniffio==1.3.0 # anyio # httpcore # httpx -sqlalchemy==2.0.17 +sqlalchemy==2.0.16 # via # -r requirements/base.in # alembic @@ -477,7 +477,7 @@ tuspy==1.0.1 # via pyvimeo tweepy==4.14.0 # via -r requirements/base.in -twilio==8.4.0 +twilio==8.3.0 # via -r requirements/base.in typing-extensions==4.6.3 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index ff2f24bac..0226b21b5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -61,7 +61,7 @@ flake8-logging-format==0.9.0 # via -r requirements/dev.in flake8-mutable==1.2.0 # via -r requirements/dev.in -flake8-plugin-utils==1.3.3 +flake8-plugin-utils==1.3.2 # via flake8-pytest-style flake8-print==5.0.0 # via -r requirements/dev.in @@ -90,7 +90,7 @@ mccabe==0.7.0 # via # flake8 # pylint -mypy==1.4.1 +mypy==1.4.0 # via -r requirements/dev.in mypy-json-report==1.0.4 # via -r requirements/dev.in @@ -106,7 +106,7 @@ pip-compile-multi==2.6.3 # via -r requirements/dev.in pip-tools==6.13.0 # via pip-compile-multi -platformdirs==3.8.0 +platformdirs==3.6.0 # via # black # pylint @@ -131,7 +131,7 @@ pyupgrade==3.7.0 # via -r requirements/dev.in reformat-gherkin==3.0.1 # via -r requirements/dev.in -ruff==0.0.275 +ruff==0.0.272 # via -r requirements/dev.in smmap==5.0.0 # via gitdb @@ -151,13 +151,13 @@ types-ipaddress==1.0.8 # via types-maxminddb types-maxminddb==1.5.0 # via types-geoip2 -types-pyopenssl==23.2.0.1 +types-pyopenssl==23.2.0.0 # via types-redis types-python-dateutil==2.8.19.13 # via -r requirements/dev.in types-pytz==2023.3.0.0 # via -r requirements/dev.in -types-redis==4.6.0.0 +types-redis==4.5.5.2 # via -r requirements/dev.in types-requests==2.31.0.1 # via -r requirements/dev.in diff --git a/requirements/test.txt b/requirements/test.txt index 5753ae509..c0dbfe9ba 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -39,11 +39,11 @@ parse==1.19.1 # pytest-bdd parse-type==0.6.0 # via pytest-bdd -pluggy==1.2.0 +pluggy==1.0.0 # via pytest py==1.11.0 # via pytest-html -pytest==7.4.0 +pytest==7.3.2 # via # -r requirements/test.in # pytest-asyncio @@ -89,7 +89,7 @@ requests-mock==1.11.0 # via -r requirements/test.in respx==0.20.1 # via -r requirements/test.in -selenium==4.10.0 +selenium==4.9.1 # via pytest-selenium sortedcontainers==2.4.0 # via trio From 9b69f71027c44c6f441ee5aa2145a0f77b7b2bcb Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 29 Jun 2023 05:02:37 +0530 Subject: [PATCH 35/45] Tags, caching and arm64 build in the last app rebuild stage --- .github/workflows/pytest-docker.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 06fcea211..28c4128fd 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -53,12 +53,24 @@ jobs: github-token: ${{ secrets.github_token }} path-to-lcov: coverage/funnel.lcov flag-name: docker-3.11 + - name: Set short git commit SHA + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + id: sha + run: | + short_sha=$(git rev-parse --short ${{ github.sha }}) + echo "::set-output name=short::$short_sha" - name: Build funnel image + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} id: build-funnel uses: docker/build-push-action@v4 with: context: . file: Dockerfile target: app - tags: funnel + platforms: linux/amd64,linux/arm64 + tags: | + hasgeek/funnel:${{ github.ref_name }} + hasgeek/funnel:sha-${{ steps.sha.outputs.short }} push: false + cache-from: type=gha + cache-to: type=gha,mode=max From d1dc0d67687075e52e6fc4ad96a2e60997f592ce Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 29 Jun 2023 05:49:09 +0530 Subject: [PATCH 36/45] QEMU is recommended for multi-platform builds; Add .github to .dockerignore --- .dockerignore | 3 +++ .github/workflows/pytest-docker.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.dockerignore b/.dockerignore index 58e88bd14..c4e0ebd32 100644 --- a/.dockerignore +++ b/.dockerignore @@ -73,6 +73,9 @@ env/ # Versioning files .git +# github workflow files +.github + # Local DBs dump.rdb test.db diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 28c4128fd..bb4c50ac9 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build funnel-ci image From 686d475439edb9ef2b84fa298123eb50c056a0d5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 29 Jun 2023 06:47:22 +0530 Subject: [PATCH 37/45] set-output is deprecated by GHA; login to registries and push docker images --- .github/workflows/pytest-docker.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index bb4c50ac9..b7c5b0efa 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -60,8 +60,21 @@ jobs: id: sha run: | short_sha=$(git rev-parse --short ${{ github.sha }}) - echo "::set-output name=short::$short_sha" - - name: Build funnel image + echo "short=$short_sha" >> "$GITHUB_OUTPUT" + - name: Login to Docker Hub + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry + if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build & push funnel image if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} id: build-funnel uses: docker/build-push-action@v4 @@ -73,6 +86,8 @@ jobs: tags: | hasgeek/funnel:${{ github.ref_name }} hasgeek/funnel:sha-${{ steps.sha.outputs.short }} - push: false + ghcr.io/hasgeek/funnel:${{ github.ref_name }} + ghcr.io/hasgeek/funnel:sha-${{ steps.sha.outputs.short }} + push: true cache-from: type=gha cache-to: type=gha,mode=max From ab3191c5d8e06c67554522bf29ba72187ec1bf8a Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 29 Jun 2023 22:01:23 +0530 Subject: [PATCH 38/45] Try personal token from https://github.com/settings/tokens/new?scopes=write:packages for publishing docker image to GHCR --- .github/workflows/pytest-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index b7c5b0efa..4511d3c1b 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -73,7 +73,7 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GHCR_WRITE_TOKEN }} - name: Build & push funnel image if: ${{ github.ref_name == 'main' || github.ref_name == 'docker' }} id: build-funnel From 8f01f2040daad0df33c8e9d1bf218efd2ac89dbd Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Thu, 29 Jun 2023 23:59:48 +0530 Subject: [PATCH 39/45] =?UTF-8?q?Remove=20arm64=20build=20from=20docker=20?= =?UTF-8?q?GHA=20workflow=20=E2=80=93=20the=20build=20process=20is=20slow,?= =?UTF-8?q?=20unstable=20and=20unreliable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest-docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 4511d3c1b..41e85b5e1 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -82,7 +82,6 @@ jobs: context: . file: Dockerfile target: app - platforms: linux/amd64,linux/arm64 tags: | hasgeek/funnel:${{ github.ref_name }} hasgeek/funnel:sha-${{ steps.sha.outputs.short }} From 3b80a858fefeb7887cda3a3f1a514d41d747ba00 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 30 Jun 2023 03:51:16 +0530 Subject: [PATCH 40/45] Update docker commands for running CI tests --- .github/workflows/pytest-docker.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest-docker.yml b/.github/workflows/pytest-docker.yml index 41e85b5e1..07752e4d6 100644 --- a/.github/workflows/pytest-docker.yml +++ b/.github/workflows/pytest-docker.yml @@ -46,7 +46,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - name: Set permissions for coverage - run: docker compose -f docker-compose.ci.yml run -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage + run: docker compose -f docker-compose.ci.yml run --rm -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage - name: Test with pytest run: docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main - name: Upload coverage report to Coveralls diff --git a/Makefile b/Makefile index 21fa3118a..36cc98ffc 100644 --- a/Makefile +++ b/Makefile @@ -49,8 +49,8 @@ docker-base-devtest: docker buildx build -f docker/images/bases.Dockerfile --target base-devtest --tag hasgeek/funnel-base-devtest . docker-ci-test: - docker compose -f docker-compose.ci.yml run -u root --entrypoint "" main chown -R 1000:1000 coverage && \ - docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --exit-code-from main + docker compose -f docker-compose.ci.yml run --rm -u root --entrypoint "" --no-deps --quiet-pull main chown -R 1000:1000 coverage && \ + docker compose -f docker-compose.ci.yml up --quiet-pull --no-attach db --no-attach redis --no-log-prefix --exit-code-from main docker-dev: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 \ From b194dd287569db889c89af121a68e645b43d37ef Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Fri, 30 Jun 2023 21:04:07 +0530 Subject: [PATCH 41/45] Added rq service in docker compose --- Dockerfile | 1 - docker-compose.yml | 6 ++++++ docker/supervisor/rq.conf | 29 +++++++++++++++++++++++++++++ docker/supervisord/supervisord.conf | 28 ---------------------------- 4 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 docker/supervisor/rq.conf delete mode 100644 docker/supervisord/supervisord.conf diff --git a/Dockerfile b/Dockerfile index 9715014e6..07519c8f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ && apt-get autoremove -yqq \ && rm -rf /var/lib/apt/lists/* \ && mkdir -pv /var/log/supervisor -COPY ./docker/supervisord/supervisord.conf /etc/supervisor/supervisord.conf RUN addgroup --gid 1000 funnel && adduser --uid 1000 --gid 1000 funnel ENV PATH "$PATH:/home/funnel/.local/bin" USER funnel diff --git a/docker-compose.yml b/docker-compose.yml index 79437fdb6..ff6d27589 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,12 @@ services: - 6410 ports: - 6410:6410 + rq: + <<: *main-app + entrypoint: ['supervisord'] + command: ['-c', 'docker/supervisor/rq.conf'] + expose: [] + ports: [] bash: <<: *main-app entrypoint: [''] diff --git a/docker/supervisor/rq.conf b/docker/supervisor/rq.conf new file mode 100644 index 000000000..f9245dbcd --- /dev/null +++ b/docker/supervisor/rq.conf @@ -0,0 +1,29 @@ +[supervisord] +nodaemon=true +logfile=/dev/null +logfile_maxbytes=0 + +[program:rq-funnel] +; Point the command to the specific rqworker command you want to run. +; If you use virtualenv, be sure to point it to +; /path/to/virtualenv/bin/rqworker +; Also, you probably want to include a settings module to configure this +; worker. For more info on that, see http://python-rq.org/docs/workers/ +command=flask rq worker funnel +process_name=%(program_name)s-%(process_num)s +user=funnel + +; If you want to run more than one worker instance, increase this +numprocs=2 + +; This is the directory from which RQ is ran. Be sure to point this to the +; directory where your source code is importable from +directory=/home/funnel/app + +; RQ requires the TERM signal to perform a warm shutdown. If RQ does not die +; within 10 seconds, supervisor will forcefully kill it +stopsignal=TERM + +; These are up to you +autostart=true +autorestart=true diff --git a/docker/supervisord/supervisord.conf b/docker/supervisord/supervisord.conf deleted file mode 100644 index afead1552..000000000 --- a/docker/supervisord/supervisord.conf +++ /dev/null @@ -1,28 +0,0 @@ -; supervisor config file - -[unix_http_server] -file=/var/run/supervisor.sock ; (the path to the socket file) -chmod=0700 ; sockef file mode (default 0700) - -[supervisord] -logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) -pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) - -; the below section must remain in the config file for RPC -; (supervisorctl/web interface) to work, additional interfaces may be -; added by defining them in separate rpcinterface: sections -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket - -; The [include] section can just contain the "files" setting. This -; setting can list multiple files (separated by whitespace or -; newlines). It can also contain wildcards. The filenames are -; interpreted as relative to this file. Included files *cannot* -; include files themselves. - -[include] -files = /etc/supervisor/conf.d/*.conf From 769338594d92d3209766c4147fdc5ca942175545 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 5 Jul 2023 07:09:47 +0530 Subject: [PATCH 42/45] Remove APP-*-SITE_ID environment variables from .testenv; Add DEBUG_TB_INTERCEPT_REDIRECTS=false in sample.env --- .testenv | 2 -- sample.env | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.testenv b/.testenv index 28914c7de..dc55878fb 100644 --- a/.testenv +++ b/.testenv @@ -46,9 +46,7 @@ FLASK_IMAGE_URL_DOMAINS='["images.example.com"]' FLASK_IMAGE_URL_SCHEMES='["https"]' FLASK_SES_NOTIFICATION_TOPICS=null # Per app config -APP_FUNNEL_SITE_ID=hasgeek-test APP_FUNNEL_SERVER_NAME=funnel.test:3002 APP_FUNNEL_SHORTLINK_DOMAIN=f.test:3002 APP_FUNNEL_DEFAULT_DOMAIN=funnel.test APP_FUNNEL_UNSUBSCRIBE_DOMAIN=bye.test -APP_SHORTLINK_SITE_ID=shortlink-test diff --git a/sample.env b/sample.env index a12e7e83c..8333161a8 100644 --- a/sample.env +++ b/sample.env @@ -24,6 +24,7 @@ FLASK_ENV=development FLASK_DEBUG=1 # Flask-DebugToolbar (optional) is useful for dev, but MUST NOT BE enabled in production FLASK_DEBUG_TB_ENABLED=true +DEBUG_TB_INTERCEPT_REDIRECTS=false # --- Domain configuration (these must point to 127.0.0.1 in /etc/hosts) # Funnel app's server name (Hasgeek uses 'hasgeek.com' in production) @@ -57,11 +58,6 @@ FLASK_SECRET_KEYS='["make-this-something-random", "older-secret-keys-here"]' FLASK_LASTUSER_SECRET_KEYS='["make-this-something-random", "older-secret-keys-here"]' # --- App configuration -# Site id is used as an identifier to disambiguate between apps as they don't have a -# name property -APP_FUNNEL_SITE_ID=hasgeek -APP_SHORTLINK_SITE_ID=shortlink -APP_UNSUBSCRIBE_SITE_ID=unsubscribe # Some templates pick up site title from this setting, but other places do not as any # value here is not localizable for translations APP_FUNNEL_SITE_TITLE=Hasgeek From cef99074d2c0e1b411553d6631c49bac642c0745 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Wed, 5 Jul 2023 08:03:45 +0530 Subject: [PATCH 43/45] Correction: Replace DEBUG_TB_INTERCEPT_REDIRECTS with FLASK_DEBUG_TB_INTERCEPT_REDIRECTS --- sample.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample.env b/sample.env index 8333161a8..0e08b2450 100644 --- a/sample.env +++ b/sample.env @@ -24,7 +24,7 @@ FLASK_ENV=development FLASK_DEBUG=1 # Flask-DebugToolbar (optional) is useful for dev, but MUST NOT BE enabled in production FLASK_DEBUG_TB_ENABLED=true -DEBUG_TB_INTERCEPT_REDIRECTS=false +FLASK_DEBUG_TB_INTERCEPT_REDIRECTS=false # --- Domain configuration (these must point to 127.0.0.1 in /etc/hosts) # Funnel app's server name (Hasgeek uses 'hasgeek.com' in production) From b5f05e03b94fd9ade5d8e7fdde0781a6ff9b10f5 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 31 Oct 2023 18:08:58 +0530 Subject: [PATCH 44/45] Install rust in docker to support installation of z-base-32 --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 07519c8f7..9f41b1b0f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN chsh -s /usr/sbin/nologin root # hadolint ignore=DL3008 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update -yqq \ - && apt-get install -yqq --no-install-recommends supervisor \ + && apt-get install -yqq --no-install-recommends supervisor curl \ && apt-get autoclean -yqq \ && apt-get autoremove -yqq \ && rm -rf /var/lib/apt/lists/* \ @@ -31,10 +31,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ RUN addgroup --gid 1000 funnel && adduser --uid 1000 --gid 1000 funnel ENV PATH "$PATH:/home/funnel/.local/bin" USER funnel +# hadolint ignore=DL4006 +RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | /bin/bash -s -- -y +ENV PATH "/home/funnel/.cargo/bin:$PATH" WORKDIR /home/funnel/app COPY --chown=funnel:funnel Makefile Makefile COPY --chown=funnel:funnel requirements/base.txt requirements/base.txt +RUN mkdir -pv /home/funnel/.cache/pip RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python COPY --chown=funnel:funnel . . From 6b99f09d9bd2c53bb688ef2b18849a796919d910 Mon Sep 17 00:00:00 2001 From: Mitesh Ashar Date: Tue, 31 Oct 2023 18:17:49 +0530 Subject: [PATCH 45/45] Install uwsgi in docker --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f41b1b0f..53ce203b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,8 @@ WORKDIR /home/funnel/app COPY --chown=funnel:funnel Makefile Makefile COPY --chown=funnel:funnel requirements/base.txt requirements/base.txt RUN mkdir -pv /home/funnel/.cache/pip -RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python +# hadolint ignore=DL3013,DL3042 +RUN --mount=type=cache,target=/home/funnel/.cache/pip,uid=1000,gid=1000 make install-python && pip install --upgrade uwsgi COPY --chown=funnel:funnel . . COPY --from=assets --chown=funnel:funnel /home/node/app/funnel/static/built/ funnel/static/build