feat: add GitHub reports for Engineering team

CONTRIBUTING.md

@@ -35,3 +35,18 @@ terraform-docs .
 ```
 
 This will replace the readme file at `deploy/infrastructure/README.md` and `deploy/meltano/README.md` with any changes made to the module and header docs.
+
+## Developing transforms
+
+To develop transforms, you'll need to duplicate `.env.template` as `.env` and ensure that at least these env vars are declared:
+
+- `SNOWFLAKE_USER`
+- `SNOWFLAKE_PASSWORD`
+
+## Incremental build with `--defer` option
+
+```console
+meltano invoke dbt-snowflake:seed
+meltano invoke dbt-snowflake:snapshot
+meltano invoke dbt-snowflake:build
+```

data/.env.template

@@ -19,10 +19,9 @@ PERMISSION_BOT_PASSWORD="****"
 PERMISSION_BOT_ACCOUNT="****"
 
 # Snowflake MELTANO User Password
+SNOWFLAKE_USER="...."
 SNOWFLAKE_PASSWORD="****"
-
-# dbt Snowflake Password
-DBT_SNOWFLAKE_PASSWORD="****"
+SNOWFLAKE_ROLE="DEVELOPER"
 
 # Slack Webhooks
 TARGET_APPRISE_SINGER_ACTIVITY_URIS=["https://hooks.slack.com/services/{}/{}/{}"]

data/.sqlfluff

@@ -17,7 +17,9 @@ profile = meltano
 tab_space_size = 4
 max_line_length = 80
 indent_unit = space
-comma_style = trailing
+
+[sqlfluff:layout:type:comma]
+line_position = trailing
 
 [sqlfluff:rules:L010] # Keywords
 capitalisation_policy = upper

data/environments/dbt_dev.meltano.yml

@@ -0,0 +1,42 @@
+environments:
+- name: dbt_dev
+  annotations:
+    docs:
+      description: >
+        Environment for developing dbt transforms separately from EL.
+  config:
+    plugins:
+      utilities:
+      - name: dbt-snowflake
+        config:
+          user: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
+          warehouse: CORE
+          skip_pre_invoke: true
+          database: USERDEV_PROD
+          database_prep: USERDEV_PREP
+          target_schema_prefix: ${SNOWFLAKE_USER}_
+      - name: sqlfluff
+        config:
+          user: ${SNOWFLAKE_USER}
+      - name: great_expectations
+        config:
+          prod_database: USERDEV_PROD
+          raw_database: USERDEV_RAW
+          username: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
+          warehouse: CORE
+  env:
+    USER_PREFIX: ${SNOWFLAKE_USER}
+    SUPERSET_API_URL: http://localhost:8088
+    SUPERSET_USER: admin
+    SUPERSET_PASS: admin
+    # https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
+    AIRFLOW__CORE__PLUGINS_FOLDER: $MELTANO_PROJECT_ROOT/orchestrate/plugins_local
+    AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: '30'
+    AIRFLOW_VAR_MELTANO_ENVIRONMENT: userdev
+    AIRFLOW_VAR_OPERATOR_TYPE: bash
+    # Secrets via KMS
+    KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
+    KMS_DOTENV_PATH: .env
+    KMS_SECRETS_PATH: secrets.yml

data/environments/userdev.meltano.yml

@@ -22,8 +22,8 @@ environments:
       - name: tap-snowflake
         config:
           dbname: USERDEV_PROD
-          user: ${USER_PREFIX}
-          role: ${USER_PREFIX}
+          user: ${SNOWFLAKE_USER}
+          role: ${SNOWFLAKE_USER}
           warehouse: CORE
       - name: tap-snowflake-metrics-legacy
         config:
@@ -85,7 +85,7 @@ environments:
           role: ${USER_PREFIX}
           warehouse: CORE
   env:
-    USER_PREFIX: PNADOLNY
+    USER_PREFIX: ${SNOWFLAKE_USER}
     SUPERSET_API_URL: http://localhost:8088
     SUPERSET_USER: admin
     SUPERSET_PASS: admin
@@ -97,4 +97,4 @@ environments:
     # Secrets via KMS
     KMS_PUBLIC_KEY_PATH: utilities/kms/Publickey.pem
     KMS_DOTENV_PATH: .env
-    KMS_SECTRETS_PATH: secrets.yml
+    KMS_SECRETS_PATH: secrets.yml

data/meltano.yml

@@ -1,5 +1,5 @@
 version: 1
-default_environment: userdev
+default_environment: dbt_dev
 send_anonymous_usage_stats: false
 project_id: c15e971a-d318-4a9d-979b-1039ce5fd1b1
 include_paths:
@@ -9,3 +9,5 @@ include_paths:
 - ./orchestrate/*.meltano.yml
 - ./transform/*.meltano.yml
 - ./utilities/*.meltano.yml
+ff:
+  strict_env_var_mode: true

data/transform/models/marts/engineering/list_issues.sql

@@ -0,0 +1,21 @@
+SELECT
+    repo_name,
+    organization_name,
+    state,
+    author_association,
+    labels,
+    reactions,
+    assignees,
+    milestone,
+    issue_id,
+    issue_number,
+    comment_count,
+    is_locked,
+    author_id,
+    author_username,
+    assignee_id,
+    assignee_username,
+    last_updated_ts,
+    created_at_ts,
+    closed_at_ts
+FROM {{ref('stg_github__issues')}}

data/transform/models/marts/engineering/schema.yml

@@ -0,0 +1,15 @@
+version: 2
+
+models:
+  - name: list_issues
+    description: One row per item, describing all github issues and PRs.
+    columns:
+      - name: issue_number
+        description: The PR or Issue number.
+        tests:
+          - not_null
+      - name: issue_id
+        description: The unique key for the issue or PR.
+        tests:
+          - not_null
+          - unique

data/transform/models/marts/telemetry/base/unstructured_parsing/event_unstruct.sql

@@ -1,3 +1,9 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+
 WITH base AS (
 
     SELECT

data/transform/models/staging/snowplow/snowplow_bad_parsed.sql

@@ -1,3 +1,9 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+
 WITH reparse_1 AS (
     -- Incident: new schema not registered to SnowcatCloud versions 2.14.0 and
     -- 2.15.0 (partial)