From 45fc87877f83ab3e98486225a91b2ae22a98ceaa Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 13 Sep 2024 14:25:02 +0545 Subject: [PATCH 1/4] Use profile in docker-compose.yml - docker compose up will only run essential services - docker compose up run --rm can be used to run cli helpers - docker compose --profile elasticsearch up is required to run kibana --- Dockerfile | 2 +- docker-compose.yml | 34 +++++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a7f69b53..82a93a5c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.9-bullseye -ENV PYTHONUNBUFFERED 1 +ENV PYTHONUNBUFFERED=1 EXPOSE 80 EXPOSE 443 diff --git a/docker-compose.yml b/docker-compose.yml index 399644376..7b904b299 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - x-server: &base_server_setup image: ifrcgo/go-api:latest build: . @@ -29,8 +27,8 @@ x-server: &base_server_setup CACHE_MIDDLEWARE_SECONDS: ${CACHE_MIDDLEWARE_SECONDS:-600} ELASTIC_SEARCH_HOST: ${ELASTIC_SEARCH_HOST:-elasticsearch://elasticsearch:9200} # Appeal API - APPEALS_USER: ${APPEALS_USER} - APPEALS_PASS: ${APPEALS_PASS} + APPEALS_USER: ${APPEALS_USER:-} + APPEALS_PASS: ${APPEALS_PASS:-} # Sentry SENTRY_DSN: ${SENTRY_DSN:-} SENTRY_SAMPLE_RATE: ${SENTRY_SAMPLE_RATE:-0.2} @@ -56,7 +54,6 @@ x-server: &base_server_setup - db - redis - elasticsearch - - kibana services: @@ -107,6 +104,7 @@ services: - '5601:5601' depends_on: - elasticsearch + profiles: [elasticsearch] serve: <<: *base_server_setup @@ -119,93 +117,119 @@ services: <<: *base_server_setup command: python manage.py run_celery_dev + # ------------------ Helper CLI Commands + # Usage: `docker compose run --rm ` + # Example: `docker compose run --rm bash` bash: <<: *base_server_setup entrypoint: /bin/bash + profiles: [cli] shell: <<: *base_server_setup command: python manage.py shell + profiles: [cli] loaddata: <<: *base_server_setup command: python manage.py loaddata Regions Countries Districts DisasterTypes Actions Groups + profiles: [cli] ingest_databank: <<: *base_server_setup command: python manage.py ingest_databank + profiles: [cli] collectstatic: <<: *base_server_setup command: python manage.py collectstatic --noinput -l + profiles: [cli] createsuperuser: <<: *base_server_setup command: python manage.py createsuperuser + profiles: [cli] ingest_appeals: <<: *base_server_setup command: python manage.py ingest_appeals + profiles: [cli] ingest_appeal_docs: <<: *base_server_setup command: python manage.py ingest_appeal_docs + profiles: [cli] user_registration_reminder: <<: *base_server_setup command: python manage.py user_registration_reminder + profiles: [cli] ingest_appeal_docs_fullscan: <<: *base_server_setup command: python manage.py ingest_appeal_docs --fullscan + profiles: [cli] ingest_mdb: <<: *base_server_setup command: python manage.py ingest_mdb + profiles: [cli] migrate: <<: *base_server_setup command: python manage.py migrate + profiles: [cli] makemigrations: <<: *base_server_setup command: python manage.py makemigrations + profiles: [cli] makemigrations_merge: <<: *base_server_setup command: python manage.py makemigrations --merge + profiles: [cli] make_permissions: <<: *base_server_setup command: python manage.py make_permissions + profiles: [cli] test: <<: *base_server_setup command: pytest --durations=10 + profiles: [cli] test_snapshot_update: <<: *base_server_setup command: python manage.py test -k --snapshot-update + profiles: [cli] testr: <<: *base_server_setup command: pytest --reuse-db --durations=10 -s + profiles: [cli] coverage: <<: *base_server_setup command: coverage run --source='.' manage.py test -k + profiles: [cli] coverage_report: <<: *base_server_setup command: coverage report + profiles: [cli] coverage_html: <<: *base_server_setup command: coverage html + profiles: [cli] triggers_to_db: <<: *base_server_setup command: python manage.py triggers_to_db + profiles: [cli] + volumes: redis-data: From 284aeae3f6ba69d85d543be423f74f320cbc085b Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 13 Sep 2024 14:27:32 +0545 Subject: [PATCH 2/4] Migrate CI from circleci to github actions --- .circleci/config.yml | 62 ----------------------------------- .github/workflows/ci.yaml | 23 ------------- .github/workflows/ci.yml | 68 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 85 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/ci.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6b60e64ee..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Python CircleCI 2.1 configuration file -version: 2.1 - -jobs: - build: - machine: - # Check https://circleci.com/developer/machine/image/ubuntu-2004 for details: - image: default - docker_layer_caching: true - steps: - - checkout - - run: - name: build docker base image - command: | - export DOCKER_BUILDKIT=1 - cp .env-sample .env - docker-compose build --progress plain serve - docker-compose pull db redis - - run: - name: Validate if there are no pending django migrations. - command: | - docker-compose run --rm serve ./manage.py makemigrations --check --dry-run || { - echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations'; - exit 1; - } - - run: - name: Validate SentryMonitor config - command: | - docker-compose run --rm serve ./manage.py cron_job_monitor --validate-only || { - echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor'; - exit 1; - } - - run: - name: Run tests - command: | - time docker-compose run --rm serve ./manage.py test --keepdb -v 2 --pattern="test_fake.py" && - docker-compose run --rm serve pytest --reuse-db --durations=10 - - run: - name: Push image to Docker Hub - command: | - if [ "${CIRCLE_BRANCH}" == "master" ] || [ "${CIRCLE_BRANCH}" == "develop" ]|| [ "${CIRCLE_BRANCH}" == "remotetest" ]; then - { - if [ "${CIRCLE_BRANCH}" == "develop" ]; then - LATEST_GIT_TAG=develop && - echo $LATEST_GIT_TAG - elif [ "${CIRCLE_BRANCH}" == "remotetest" ]; then - LATEST_GIT_TAG=remotetest && - echo $LATEST_GIT_TAG - else - LATEST_GIT_TAG=$(git describe --tags --abbrev=0 --match 1.*) && - echo $LATEST_GIT_TAG - git checkout tags/$LATEST_GIT_TAG -b latest_git_tag - fi - - docker-compose build - docker login -u $DOCKER_USER -p $DOCKER_PASS - docker tag ifrcgo/go-api:latest ifrcgo/go-api:$LATEST_GIT_TAG - docker push ifrcgo/go-api:$LATEST_GIT_TAG - } || { - echo 'No tag matching the condition found' - } - fi diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index b024411da..000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Python check - -on: - push: - branches: - - develop - pull_request: - - -jobs: - pre_commit_checks: - name: ๐Ÿšด Pre-Commit checks ๐Ÿšด - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@main - - name: Install poetry - run: pipx install poetry - - uses: actions/setup-python@main - with: - cache: 'poetry' - - run: poetry install - - uses: pre-commit/action@main diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..22bdc909f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: CI + +on: + workflow_call: + pull_request: + # NOTE: For develop & master, they are run through helm github action ./build-publish-docker-helm.yml + + +jobs: + pre_commit_checks: + name: ๐Ÿšด Pre-Commit checks ๐Ÿšด + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@main + - name: Install poetry + run: pipx install poetry + - uses: actions/setup-python@main + with: + cache: 'poetry' + - run: poetry install + - uses: pre-commit/action@main + + test: + name: ๐Ÿšด Checks & Test ๐Ÿšด + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@main + + - name: ๐Ÿณ Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: ๐Ÿณ Build image + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + push: false + load: true + tags: ifrcgo/go-api:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Docker config setup & Pull docker images + run: | + cp .env-sample .env && + docker compose run --rm serve ls + + - name: ๐Ÿ•ฎ Validate if there are no pending django migrations. + run: | + docker compose run --rm serve ./manage.py makemigrations --check --dry-run || { + echo 'There are some changes to be reflected in the migration. Make sure to run makemigrations'; + exit 1; + } + + - name: ๐Ÿ•ฎ Validate SentryMonitor config + run: | + docker compose run --rm serve ./manage.py cron_job_monitor --validate-only || { + echo 'There are some changes to be reflected in the SentryMonitor. Make sure to update SentryMonitor'; + exit 1; + } + + - name: ๐Ÿคž Run Test ๐Ÿงช + run: | + time docker compose run --rm serve ./manage.py test --keepdb -v 2 --pattern="test_fake.py" && + docker compose run --rm serve pytest --reuse-db --durations=10 From c6c6486212527a25af7d20dea14ad2add5cff38d Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 13 Sep 2024 16:10:29 +0545 Subject: [PATCH 3/4] Trigger CI before build-publish-docker-helm.yml --- ...ish-docker-helm.yaml => build-publish-docker-helm.yml} | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename .github/workflows/{build-publish-docker-helm.yaml => build-publish-docker-helm.yml} (91%) diff --git a/.github/workflows/build-publish-docker-helm.yaml b/.github/workflows/build-publish-docker-helm.yml similarity index 91% rename from .github/workflows/build-publish-docker-helm.yaml rename to .github/workflows/build-publish-docker-helm.yml index 7312357e8..b5b9c4b63 100644 --- a/.github/workflows/build-publish-docker-helm.yaml +++ b/.github/workflows/build-publish-docker-helm.yml @@ -14,7 +14,13 @@ permissions: packages: write jobs: + ci: + name: CI + uses: ./.github/workflows/ci.yml + build: + name: Publish Helm + needs: ci runs-on: ubuntu-latest steps: @@ -55,4 +61,4 @@ jobs: - name: Push Helm Chart run: | - helm push .helm-charts/ifrcgo-helm-${{ steps.get_version.outputs.VERSION }}.tgz oci://ghcr.io/ifrcgo/go-api \ No newline at end of file + helm push .helm-charts/ifrcgo-helm-${{ steps.get_version.outputs.VERSION }}.tgz oci://ghcr.io/ifrcgo/go-api From 761fdbbb1b2cb4432554283bf63307eb8ae4ee32 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Fri, 13 Sep 2024 16:19:17 +0545 Subject: [PATCH 4/4] Run django migration as separate job run We need to squash our migrations, it takes around 3min --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22bdc909f..2ead21698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,8 @@ jobs: exit 1; } + - name: Run django migrations + run: docker compose run --rm serve ./manage.py test --keepdb -v 2 --pattern="test_fake.py" + - name: ๐Ÿคž Run Test ๐Ÿงช - run: | - time docker compose run --rm serve ./manage.py test --keepdb -v 2 --pattern="test_fake.py" && - docker compose run --rm serve pytest --reuse-db --durations=10 + run: docker compose run --rm serve pytest --reuse-db --durations=10