CI test

Add CI test and lint pipeline

.dockerignore

@@ -1,8 +1,6 @@
-.git
-.*
-!.flake8
-~*
-db
-docs
-dataset
-cv2dnn
+*
+!docker/bin
+!docker/conf
+!pyproject.toml
+!pdm.lock
+!src

.github/workflows/ci.yml

@@ -7,193 +7,130 @@ on:
   pull_request:
     branches:
     - develop
+env:
+  HASH_SEEDS: pdm.lock docker/bin/* docker/conf/* docker/Dockerfile
+  DOCKER_CACHE_IMAGE: ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images
+  BUILD_DATE: $(date +"%Y-%m-%d %H:%M")
+  DOCKER_DEFAULT_PLATFORM: linux/amd64
 
 jobs:
-    build_and_push_dev:
-        runs-on: ubuntu-latest
-        steps:
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v1
-          - name: Checkout code
-            uses: actions/checkout@v2
-          - name: DockerHub login
-            uses: docker/login-action@v1
-            with:
-              username: ${{ secrets.DOCKERHUB_USERNAME }}
-              password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-          - name: Push dev
-            run: |
-              docker buildx create --use
-              docker buildx build \
-                --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-${{ github.sha }}-dev \
-                --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-latest-dev \
-                --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-${{ github.sha }}-dev \
-                --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-latest-dev \
-                -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}-dev \
-                -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-latest-dev \
-                -f ./docker/Dockerfile \
-                --target dev \
-                --push \
-                ./
-                
-    black:
-      runs-on: ubuntu-latest
-      needs: [build_and_push_dev]
+    build:
+      runs-on: ubuntu-20.04
       steps:
         - name: Checkout code
-          uses: actions/checkout@v2
-        - name: DockerHub login
-          uses: docker/login-action@v1
-          with:
-            username: ${{ secrets.DOCKERHUB_USERNAME }}
-            password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-        - name: Check
-          run: |
-            docker run --rm -i \
-              ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}-dev \
-              black . --check
-
-    flake8:
-      runs-on: ubuntu-latest
-      needs: [build_and_push_dev]
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v2
+          uses: actions/checkout@v4
         - name: DockerHub login
-          uses: docker/login-action@v1
+          uses: docker/login-action@v3
           with:
             username: ${{ secrets.DOCKERHUB_USERNAME }}
             password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-        - name: Check
+        - name: Check Dev Image
+          continue-on-error: true
+          id: image_exists
           run: |
-            docker run --rm -i \
-              ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}-dev \
-              flake8 \
-              --max-complexity 12 \
-              --max-line-length 120 \
-              --exclude=.*/,__pycache__,docs,~build,dist,*.md \
-              --per-file-ignores='src/**/migrations/*.py:E501' \
-              .
-              
-    unit_tests:
-      runs-on: ubuntu-latest
-      needs: [build_and_push_dev]
-      steps:
-        - name: Checkout code
-          uses: actions/checkout@v2
-        - name: DockerHub login
-          uses: docker/login-action@v1
-          with:
-            username: ${{ secrets.DOCKERHUB_USERNAME }}
-            password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-        - name: Unit tests
-          run: |
-            backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}-dev docker compose \
-              -f ./ops/compose.ci-test.yml \
-              up --exit-code-from backend
-
-    build_and_push_prd:
-      needs: [build_and_push_dev]
-      runs-on: ubuntu-latest
-      steps:
-        - name: Set up Docker Buildx
-          uses: docker/setup-buildx-action@v1
-        - name: Checkout code
-          uses: actions/checkout@v2
-        - name: DockerHub login
-          uses: docker/login-action@v1
-          with:
-            username: ${{ secrets.DOCKERHUB_USERNAME }}
-            password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-        - name: Push prd
+            set +e
+            LOCK_SHA=$(echo sha1sum ${{env.HASH_SEEDS}}| sha1sum | awk '{print $1}' | cut -c 1-8)
+            exists=$(docker manifest inspect ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} > /dev/null 2>&1 && echo "exists" || echo "not_found")
+            exists="exists"
+            echo "result=$exists" >> "$GITHUB_OUTPUT"
+            echo "SHA=$LOCK_SHA" >> "$GITHUB_OUTPUT"
+            if [ $exists = "exists" ];then
+              echo "Tag ${{ steps.image_exists.outputs.SHA }} found"
+            else
+              echo "Tag ${{ steps.image_exists.outputs.SHA }} does not exist"            
+            fi
+        - name: Build Dev Image
+          if: ${{ steps.image_exists.outputs.result == 'not_found' || contains(github.event.head_commit.message, 'rebuild') }}
           run: |
-            docker buildx create --use
-
-            # Base part of the command
-            build_command="docker buildx build \
+            LOCK_SHA=${{ steps.image_exists.outputs.SHA }}
+            docker buildx create --use --platform x86_64 --name builder --driver docker-container
+            docker buildx build \            
+              --platform x86_64 \
+              --builder builder \
+              --build-arg BUILD_DATE="${BUILD_DATE}" \
               --progress=plain \
-              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-${{ github.sha }}-dev \
-              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-latest-dev \
-              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-${{ github.sha }}-prd \
-              --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-latest-prd \
-              --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-${{ github.sha }}-prd \
-              --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-deduplication-engine-latest-prd \
-              -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}-prd \
-              -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }} \
+              --cache-to type=local,ref=${DOCKER_CACHE_IMAGE}:hde-dev-latest,dest=./.AAA \
+              --cache-from ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
+              -t ${DOCKER_CACHE_IMAGE}:hde-dev-${LOCK_SHA} \
+              -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
               -f ./docker/Dockerfile \
-              --target prd \
-              --push ./"
-            
-            if [ "${{ github.ref }}" = "refs/heads/master" ]; then
-              version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))")
-              tagged_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope:deduplication-engine-$version
-              build_command="$build_command -t $tagged_image"
-            fi
-            
-            eval $build_command
+              --push \
+              --target python_dev_deps .
 
-    trivy:
-      runs-on: ubuntu-latest
-      needs: [build_and_push_prd]
+    lint:
+      runs-on: ubuntu-20.04
+      needs: [build]
+      container:
+        image: unicef/hope-support-images:hde-dev-latest
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       steps:
         - name: Checkout code
           uses: actions/checkout@v2
-        - name: DockerHub login
-          uses: docker/login-action@v1
-          with:
-            username: ${{ secrets.DOCKERHUB_USERNAME }}
-            password: ${{ secrets.DOCKERHUB_TOKEN }}
-        - name: Run Trivy vulnerability scanner
-          uses: aquasecurity/trivy-action@master
-          with:
-            image-ref: '${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:deduplication-engine-${{ github.sha }}'
-            format: 'table'
-            exit-code: '0'
-            ignore-unfixed: true
-            vuln-type: 'os,library'
-            severity: 'CRITICAL,HIGH'
+        - name: Black
+          run: black --check src/
+        - name: Flake8
+          run: flake8 src/
 
-    deploy:
-      runs-on: ubuntu-latest
-      needs: [unit_tests, black, flake8, build_and_push_prd]
-      if: |
-        github.event_name == 'push' &&
-        (
-          github.ref == 'refs/heads/develop'
-        )
+    test:
+      runs-on: ubuntu-20.04
+      needs: [build]
+      container:
+        image: unicef/hope-support-images:hde-dev-latest
+        credentials:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+        ports:
+          - 8000:8000
+      services:
+        redis:
+          image: redis
+        db:
+          image: postgres:14
+          env:
+            POSTGRES_DATABASE: dedupe
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_USERNAME: postgres
+          options: >-
+            --health-cmd pg_isready
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+      env:
+        DATABASE_URL: postgres://postgres:postgres@db:5432/dedupe
+        SECRET_KEY: secret_key
+        CACHE_URL: redis://redis:6379/0
+        CELERY_BROKER_URL: redis://redis:6379/0
+        PYTHONPATH: "/hde/code/src:/hde/__pypackages__/3.12/lib"
       steps:
-        - name: Trigger deploy
+        - name: Checkout code
+          uses: actions/checkout@v2
+        - name: Run tests
           run: |
-            if [ ${{ github.ref }} == 'refs/heads/develop' ]; then
-              pipelineId=1309
-            else
-              echo "No pipeline to trigger for ref ${{ github.ref }}"
-              exit 0
-            fi
-
-            IFS=',' read -ra pipelines <<< "$pipelineId"
-            for pipeline in "${pipelines[@]}"; do
-              jsonBody='{"variables": {"sha": {"isSecret": false, "value": "${{ github.sha }}"}, "tag": {"isSecret": false, "value": "deduplication-engine-${{ github.sha }}"}}}'
-              contentLength=$(echo -n $jsonBody | wc -c)
-              project=ICTD-HCT-MIS
-              organization=unicef
-  
-              echo Triggering deploy for pipeline $pipeline
-              echo JSON body: $jsonBody
-
-              curl -f -v -L \
-                -u ":${{ secrets.AZURE_PAT }}" \
-                -H "Content-Type: application/json" \
-                -H "Content-Length: $contentLength" \
-                -d "$jsonBody" \
-                https://dev.azure.com/$organization/$project/_apis/pipelines/$pipeline/runs?api-version=7.1-preview.1
-              if [ $? -ne 0 ]; then
-                echo "Failed to trigger deploy for pipeline $pipeline"
-                exit 1
-              fi
-            done
+            pytest tests
+#              echo "==================================="
+#              ls -al
+#                docker run --rm \
+#                -e PYTHONPATH=/hde/code/src:/hde/__pypackages__/3.12/lib \
+#                -e CACHE_URL="${CACHE_URL}" \
+#                -e DATABASE_URL="${DATABASE_URL}" \
+#                -e SECRET_KEY="${SECRET_KEY}" \
+#                -e CELERY_BROKER_URL="${CELERY_BROKER_URL}" \
+#                -v ${PWD}:/hde/code/ \
+#                -w /hde/code/ \
+#                -t ${DOCKER_CACHE_IMAGE}:hde-dev-latest \
+#                pytest tests/ --create-db -v --cov --cov-report xml:coverage.xml
+#
+#        - name: Upload coverage to Codecov
+#          uses: codecov/codecov-action@v4
+#          with:
+#            directory: ./coverage/reports/
+#            env_vars: OS,PYTHON
+#            fail_ci_if_error: true
+#            files: /hde/code/coverage1.xml
+#            flags: unittests
+#            name: codecov-umbrella
+#            token: ${{ secrets.CODECOV_TOKEN }}
+#            verbose: true

.gitignore

@@ -15,10 +15,14 @@ __pycache__/
 !.pre-commit-config.yaml
 !.bumpversion.cfg
 !.trivyignore
+!docker/bin/*.sh
 !bandit.yaml
+build
+dist
 coverage.xml
 Makefile
 site
 black.txt
 flake8
-.env
+act.*
+

CHANGELOG.rst

@@ -0,0 +1,4 @@
+v.0.1.0 2024-05-26
+==================
+
+No significant changes.