ci.yml

name: Data Engineering CI
on:
    push:
        branches:
            - main
            - feat/*
            - fix/*
    workflow_dispatch:

jobs:
    run-ci:
        runs-on: ubuntu-latest
        env:
          WHL_GDRIVE_ID: ${{ secrets.WHL_GDRIVE_ID }}
        
        steps:
            - name: Checkout repository
              uses: actions/checkout@v4

            - name: Install dependencies
              run: |
                python3 -m pip install --upgrade pip
                pip3 install -r requirements.txt
            
            - name: Download & Install awsglue whl
              run: |
                curl -L -o aws_glue_libs-4.0.0-py3-none-any.whl "https://drive.google.com/uc?export=download&id=$WHL_GDRIVE_ID"
                pip3 install ./aws_glue_libs-4.0.0-py3-none-any.whl

            - name: Type check
              run: mypy ./ --ignore-missing-imports

            - name: Lint
              run: |
                pylint app tests jobs setup.py
                pylint app tests jobs setup.py --output pylint-report.txt

            - name: Testing
              run: |
                export KAGGLE_KEY=MOCKKEY
                export KAGGLE_USERNAME=MOCKUSERNAME
                coverage run --source=app -m unittest discover -s tests

            - name: Test coverage report
              run: |
                coverage report
                coverage xml

            - name: SonarQube Scan
              uses: sonarsource/sonarqube-scan-action@master
              with:
                args: >
                  -Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
              env:
                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
                SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}

            - uses: sonarsource/sonarqube-quality-gate-action@master
              timeout-minutes: 5
              env:
                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
                SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}