feat: configure batch pipelines #539
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Meltano Pipelines | |
| on: | |
| pull_request: | |
| branches: [ "main" ] | |
| workflow_dispatch: | |
| jobs: | |
| airflow_tests: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Airflow | |
| - run: meltano install utility airflow | |
| - run: meltano test airflow | |
| sqlfluff_tests: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Add SSH key for accessing private dbt package repo | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }} | |
| # Run Test | |
| - run: meltano install utilities sqlfluff dbt-snowflake | |
| - run: meltano run dbt-snowflake:deps | |
| - run: meltano invoke sqlfluff lint | |
| # - run: meltano invoke sqlfluff lint $(python sqlfluff_file_changes.py) | |
| dbt_seed: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Add SSH key for accessing private dbt package repo | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }} | |
| # Run Test | |
| - run: meltano install utility dbt-snowflake | |
| - run: meltano run dbt-snowflake:run_dbt_artifacts dbt-snowflake:seed | |
| el_tests: | |
| runs-on: ubuntu-latest | |
| needs: [airflow_tests, sqlfluff_tests, dbt_seed] | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| strategy: | |
| matrix: | |
| extract-source: ['meltanohub_el', 'spreadsheets_anywhere_el', 'github_search_el', 'github_meltano_el', 'gitlab_el', 'slack_el', 'google_analytics_el', 'snowplow_el', 'cloudwatch_el', 'dynanmodb_el'] | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "DEFAULT_START_DATE_1D=$(date +'%Y-%m-%d' -d '1 day ago')" >> $GITHUB_ENV | |
| - run: echo "DEFAULT_START_DATE_24HR=$(date +'%Y-%m-%d %H:%M:%S' -d '24 hours ago')" >> $GITHUB_ENV | |
| - run: echo "DEFAULT_START_DATE_4HR=$(date +'%Y-%m-%d %H:%M:%S' -d '4 hours ago')" >> $GITHUB_ENV | |
| - run: echo "DEFAULT_START_DATE_2HR=$(date +'%Y-%m-%d %H:%M:%S' -d '2 hours ago')" >> $GITHUB_ENV | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Add SSH key for accessing private dbt package repo | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }} | |
| # Install Plugins | |
| - run: meltano install --schedule=${{ matrix.extract-source }} | |
| # Run Test | |
| - run: meltano run ${{ matrix.extract-source }} | |
| transform_tests: | |
| runs-on: ubuntu-latest | |
| needs: [el_tests] | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Add SSH key for accessing private dbt package repo | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }} | |
| # Run Test | |
| - run: meltano install utility dbt-snowflake | |
| - run: meltano run dbt-snowflake:run dbt-snowflake:test | |
| reverse_el_tests: | |
| runs-on: ubuntu-latest | |
| needs: [transform_tests] | |
| defaults: | |
| run: | |
| working-directory: data/ | |
| environment: test | |
| env: | |
| MELTANO_ENVIRONMENT: cicd | |
| MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' | |
| CI_BRANCH: 'b${{ github.SHA }}' | |
| TAP_SNOWFLAKE_METRICS_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS"]' | |
| TAP_SNOWFLAKE_AUDIT_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.HUB_METRICS_AUDIT"]' | |
| TAP_SNOWFLAKE_SINGER_ACTIVITY: '["b${{ github.SHA }}_MELTANO_HUB.SINGER_ACTIVITY_NOTIFICATIONS"]' | |
| TAP_SNOWFLAKE_MELTANO_ACTIVITY: '["b${{ github.SHA }}_MELTANO_HUB.MELTANO_ACTIVITY_NOTIFICATIONS"]' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| cache: 'pip' | |
| - run: pip install -r requirements.txt | |
| - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env | |
| # Add SSH key for accessing private dbt package repo | |
| - uses: webfactory/[email protected] | |
| with: | |
| ssh-private-key: ${{ secrets.GIT_SSH_PRIVATE_KEY }} | |
| # Install Plugins | |
| - run: meltano install --schedule=hub_metrics_publish | |
| # Run Test | |
| - run: meltano run hub_metrics_publish |