bigquery.yml

name: Bigquery CI/CD

on:
  push:
    branches:
      - main
    paths:
      - "clouds/bigquery/**"
      - ".github/workflows/bigquery.yml"
  pull_request:
    paths:
      - "clouds/bigquery/**"
      - ".github/workflows/bigquery.yml"
  workflow_dispatch:
  workflow_call:

env:
  NODE_VERSION: 14
  PYTHON3_VERSION: 3.8.10
  VIRTUALENV_VERSION: 20.21.1
  GCLOUD_VERSION: 500.0.0

jobs:

  test:
    runs-on: ubuntu-20.04
    timeout-minutes: 30
    env:
      BQ_PREFIX: ci_${{ github.sha }}_${{ github.run_id }}_${{ github.run_attempt }}_
      BQ_PROJECT: ${{ secrets.BQ_PROJECT_CI }}
      BQ_BUCKET: ${{ secrets.BQ_BUCKET_CI }}
      BQ_REGION: ${{ secrets.BQ_REGION_CI }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Check diff
        uses: technote-space/get-diff-action@v4
      - name: Setup node
        uses: actions/setup-node@v1
        with:
          node-version: ${{ env.NODE_VERSION }}
      - name: Auth google
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.BQCARTOCI_DEPLOY_CLOUD_EXTENSIONS_SA_BASE64 }}
          project_id: ${{ env.BQ_PROJECT_CI }}
          create_credentials_file: true
      - name: Setup gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
            version: ${{ env.GCLOUD_VERSION }}
            project_id: ${{ env.BQ_PROJECT_CI }}
      - name: Setup python 3
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.PYTHON3_VERSION }}
      - name: Setup virtualenv
        run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
      - name: Run linter and tests
        run: |
          cd clouds/bigquery
          make lint && \
          make deploy diff="$GIT_DIFF" && \
          make test diff="$GIT_DIFF" && \
          make remove

  deploy-internal:
    if: github.ref_name == 'main'
    needs: test
    runs-on: ubuntu-20.04
    timeout-minutes: 20
    strategy:
      matrix:
        include:
          - project: BQ_PROJECT_CD
            bucket: BQ_BUCKET_CD
            region: BQ_REGION_CD
    env:
      BQ_PROJECT: ${{ secrets[matrix.project] }}
      BQ_BUCKET: ${{ secrets[matrix.bucket] }}
      BQ_REGION: ${{ secrets[matrix.region] }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Check diff
        uses: technote-space/get-diff-action@v4
      - name: Auth google
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.BQCARTOCD_DEPLOY_CLOUD_EXTENSIONS_SA_BASE64 }}
          project_id: ${{ env.BQ_PROJECT }}
          create_credentials_file: true
      - name: Setup gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
            version: ${{ env.GCLOUD_VERSION }}
            project_id: ${{ env.BQ_PROJECT }}
      - name: Run deploy
        run: |
          cd clouds/bigquery
          make deploy diff="$GIT_DIFF" production=1

  deploy:
    if: github.ref_name == 'stable'
    needs: test
    runs-on: ubuntu-20.04
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        include:
          - region: us
            project: carto-os
            bucket: gs://spatialextension_os
            permissions: '[{"specialGroup":"allAuthenticatedUsers","role":"READER"}]'
          - region: eu
            project: carto-os-eu
            bucket: gs://spatialextension_os
            permissions: '[{"specialGroup":"allAuthenticatedUsers","role":"READER"}]'
    env:
      BQ_REGION: ${{ matrix.region }}
      BQ_PROJECT: ${{ matrix.project }}
      BQ_BUCKET: ${{ matrix.bucket }}
      BQ_PERMISSIONS: ${{ matrix.permissions }}
      BQ_PERMISSIONS_GROUP: ${{ matrix.group }}
      BQ_PERMISSIONS_ROLE_NAME: spatialextensionUser
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Check diff
        uses: technote-space/get-diff-action@v4
      - name: Auth google
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.BQCARTO_DEPLOY_CLOUD_EXTENSIONS_SA_BASE64 }}
          project_id: ${{ matrix.project }}
          create_credentials_file: true
      - name: Setup gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
            version: ${{ env.GCLOUD_VERSION }}
            project_id: ${{ matrix.project }}
      - name: Run deploy
        run: |
          cd clouds/bigquery
          make deploy diff="$GIT_DIFF" production=1

  publish:
    if: github.ref_name == 'stable'
    needs: test
    runs-on: ubuntu-20.04
    timeout-minutes: 10
    env:
      PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/bigquery
      BQ_BUCKET: 'gs://@@BUCKET@@'
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
      - name: Auth google
        uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCLOUD_PRODUCTION_RELEASE_SA }}
          project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
          create_credentials_file: true
      - name: Setup gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
            version: ${{ env.GCLOUD_VERSION }}
            project_id: ${{ secrets.GCLOUD_PRODUCTION_PROJECT }}
      - name: Create package
        run: |
          cd clouds/bigquery
          make create-package
      - name: Publish package
        run: |
          cd clouds/bigquery
          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/
          gsutil -h "Content-Type:application/zip" -m cp dist/*.zip ${{ env.PACKAGE_BUCKET }}/carto-analytics-toolbox-core-bigquery-latest.zip
          gsutil -h "Content-Type:application/json" -m cp dist/*.json ${{ env.PACKAGE_BUCKET }}/