cicd.meltano.yml

environments:
- name: cicd
  config:
    plugins:
      extractors:
      - name: tap-cloudwatch
        config:
          start_date: ${DEFAULT_START_DATE_2HR}
      - name: tap-slack
        config:
          start_date: ${DEFAULT_START_DATE_4HR}
      - name: tap-slack-public
        config:
          start_date: ${DEFAULT_START_DATE_4HR}
      - name: tap-google-analytics
        config:
          start_date: ${DEFAULT_START_DATE_1D}
      - name: tap-gitlab
        config:
          start_date: ${DEFAULT_START_DATE_24HR}
      - name: tap-github
        config:
          start_date: ${DEFAULT_START_DATE_24HR}
      - name: tap-github-meltano
        config:
          start_date: ${DEFAULT_START_DATE_24HR}
      - name: tap-github-search
        config:
          start_date: ${DEFAULT_START_DATE_24HR}
          searches:
          # In CI we dont search forks and limit orgs to save time and requests
          - name: tap non-forks
            query: tap- fork:false language:Python singer in:readme org:MeltanoLabs
              org:transferwise
          - name: target non-forks
            query: target- fork:false language:Python singer in:readme org:MeltanoLabs
              org:transferwise
      # Reverse ETL connections:
      - name: tap-snowflake
        config:
          database: CICD_PROD
          user: CICD
          role: CICD
          warehouse: CICD
      # https://github.com/meltano/meltano/issues/3171 is blocking this templating from working
      - name: tap-snowflake-metrics
        config:
          tables:
          - ${CI_BRANCH}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS
      - name: tap-snowflake-audit
        config:
          tables:
          - ${CI_BRANCH}_MELTANO_HUB.HUB_METRICS_AUDIT
      - name: tap-snowflake-singer-activity
        config:
          tables:
          - ${CI_BRANCH}_SLACK_NOTIFICATIONS.SINGER_ACTIVITY_NOTIFICATIONS
      - name: tap-snowflake-meltano-activity
        config:
          tables:
          - ${CI_BRANCH}_SLACK_NOTIFICATIONS.MELTANO_ACTIVITY_NOTIFICATIONS
      - name: tap-snowflake-hubspot-companies
        config:
          tables:
          - ${CI_BRANCH}_HUBSPOT.ORG_ACTIVITY
          stream_maps:
            # TODO: This wont actually work until the nested env vars issue is fixed
            ${CI_BRANCH}_hubspot-org_activity:
              org_first_active_date: str(record.get('org_first_active_date')[:10])
                if record.get('org_first_active_date') else None
              org_last_active_date: str(record.get('org_last_active_date')[:10]) if
                record.get('org_last_active_date') else None

      loaders:
      - name: target-snowflake
        config:
          database: CICD_RAW
          user: CICD
          role: CICD
          warehouse: CICD
          default_target_schema: ${CI_BRANCH}_${MELTANO_EXTRACT__LOAD_SCHEMA}
      utilities:
      - name: dbt-snowflake
        config:
          user: CICD
          role: CICD
          warehouse: CICD
          database: CICD_PROD
          database_prep: CICD_PREP
          database_raw: CICD_RAW
          source_schema_prefix: ${CI_BRANCH}_
          target_schema_prefix: ${CI_BRANCH}_
      - name: sqlfluff
        config:
          user: CICD
          role: CICD
          warehouse: CICD
      - name: great_expectations
        config:
          prod_database: CICD_PROD
          raw_database: CICD_RAW
          username: CICD
          role: CICD
          warehouse: CICD
  env:
    # Required to be set in environment
    # DEFAULT_START_DATE_1D: "2023-02-10 01:00:00"
    AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local
    AIRFLOW_VAR_OPERATOR_TYPE: bash