diff --git a/data/github_actions__runtime_detection.csv b/data/github_actions__runtime_detection.csv new file mode 100644 index 00000000..ae095e17 --- /dev/null +++ b/data/github_actions__runtime_detection.csv @@ -0,0 +1,8 @@ +workflow_name +dbt_run_abi_refresh +dbt_run_dev_refresh +dbt_run_operation_reorg +dbt_run_scheduled_curated +dbt_run_scheduled_non_realtime +dbt_run_streamline_chainhead +dbt_run_streamline_decoder \ No newline at end of file diff --git a/models/github_actions/github_actions__current_workflows.sql b/models/github_actions/github_actions__current_workflows.sql new file mode 100644 index 00000000..f63c4d97 --- /dev/null +++ b/models/github_actions/github_actions__current_workflows.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +SELECT + NAME, + status, + created_at, + updated_at, + run_started_at, + run_attempt, + run_number, + TIMESTAMPDIFF(seconds, run_started_at, SYSDATE()) / 60 AS run_minutes, + id, + workflow_id, + html_url +FROM + TABLE( + github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'status' :'in_progress' } + ) + ) +WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }} + ) diff --git a/models/github_actions/github_actions__current_workflows.yml b/models/github_actions/github_actions__current_workflows.yml new file mode 100644 index 00000000..ed95f8b3 --- /dev/null +++ b/models/github_actions/github_actions__current_workflows.yml @@ -0,0 +1,8 @@ +version: 2 +models: + - name: github_actions__current_workflows + columns: + - name: RUN_MINUTES + tests: + - dbt_expectations.expect_column_values_to_be_between: + max_value: 59 \ No newline at end of file diff --git a/models/github_actions/github_actions__workflow_history.sql b/models/github_actions/github_actions__workflow_history.sql new file mode 100644 index 00000000..5781d5f5 --- /dev/null +++ b/models/github_actions/github_actions__workflow_history.sql @@ -0,0 +1,132 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'workflow_history_id', + tags = ['gha_tasks'], + full_refresh = false +) }} + +WITH + +{% if is_incremental() %} +-- get latest +workflow_runs AS ( + + SELECT + id AS run_id, + NAME, + node_id, + check_suite_id, + check_suite_node_id, + head_branch, + head_sha, + run_number, + event, + display_title, + status, + conclusion, + workflow_id, + url, + html_url, + pull_requests, + created_at, + updated_at, + actor, + run_attempt, + run_started_at, + triggering_actor, + jobs_url, + logs_url, + check_suite_url, + artifacts_url, + cancel_url, + rerun_url, + workflow_url, + head_commit, + repository, + head_repository + FROM + TABLE( + github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'per_page' :'100', + 'page': '1' } + ) + ) + WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }} + ) + AND conclusion IS NOT NULL +) +{% else %} + -- get history, last 100 pages +workflow_runs AS ({% for item in range(100) %} + ( + SELECT + id AS run_id, + NAME, + node_id, + check_suite_id, + check_suite_node_id, + head_branch, + head_sha, + run_number, + event, + display_title, + status, + conclusion, + workflow_id, + url, + html_url, + pull_requests, + created_at, + updated_at, + actor, + run_attempt, + run_started_at, + triggering_actor, + jobs_url, + logs_url, + check_suite_url, + artifacts_url, + cancel_url, + rerun_url, + workflow_url, + head_commit, + repository, + head_repository + FROM + TABLE(github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'per_page' :'100', + 'page': '{{ item }}' } + ) + ) + WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }}) + AND conclusion IS NOT NULL) + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %}) +{% endif %} + +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['run_id'] + ) }} AS workflow_history_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + workflow_runs qualify (ROW_NUMBER() over (PARTITION BY run_id +ORDER BY + created_at DESC)) = 1