From 1c9184b158ce25d79bee65a90d8154bd07f5d36b Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 30 Apr 2024 14:57:49 -0600 Subject: [PATCH 1/3] where filter --- .../github_actions__wf_in_progress.sql | 50 +++++++++++++++++++ .../github_actions__wf_in_progress.yml | 8 +++ 2 files changed, 58 insertions(+) create mode 100644 models/github_actions/github_actions__wf_in_progress.sql create mode 100644 models/github_actions/github_actions__wf_in_progress.yml diff --git a/models/github_actions/github_actions__wf_in_progress.sql b/models/github_actions/github_actions__wf_in_progress.sql new file mode 100644 index 00000000..6a0869a3 --- /dev/null +++ b/models/github_actions/github_actions__wf_in_progress.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'view', + tags = ['gha_workflows'] +) }} + +SELECT + id, + NAME, + node_id, + check_suite_id, + check_suite_node_id, + head_branch, + head_sha, + run_number, + event, + display_title, + status, + conclusion, + workflow_id, + url, + html_url, + pull_requests, + created_at, + updated_at, + actor, + run_attempt, + run_started_at, + triggering_actor, + jobs_url, + logs_url, + check_suite_url, + artifacts_url, + cancel_url, + rerun_url, + workflow_url, + head_commit, + repository, + head_repository, + TIMESTAMPDIFF(seconds, run_started_at, SYSDATE()) / 60 AS delta_minutes +FROM + TABLE( + github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'status' :'in_progress' } + ) + ) +WHERE + NAME NOT ILIKE '%history%' + AND NAME NOT ILIKE '%test%' + AND NAME NOT ILIKE '%heal%' diff --git a/models/github_actions/github_actions__wf_in_progress.yml b/models/github_actions/github_actions__wf_in_progress.yml new file mode 100644 index 00000000..8fd8e7f0 --- /dev/null +++ b/models/github_actions/github_actions__wf_in_progress.yml @@ -0,0 +1,8 @@ +version: 2 +models: + - name: github_actions__wf_in_progress + columns: + - name: DELTA_MINUTES + tests: + - dbt_expectations.expect_column_values_to_be_between: + max_value: 59 \ No newline at end of file From 59cf97e2f3d05dd15f53c0e5c519f798d796a94e Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 13 May 2024 14:03:48 -0600 Subject: [PATCH 2/3] add workflow history and seed file --- data/github_actions__runtime_detection.csv | 8 ++ .../github_actions__current_workflows.sql | 31 ++++ ... => github_actions__current_workflows.yml} | 4 +- .../github_actions__wf_in_progress.sql | 50 ------- .../github_actions__workflow_history.sql | 132 ++++++++++++++++++ 5 files changed, 173 insertions(+), 52 deletions(-) create mode 100644 data/github_actions__runtime_detection.csv create mode 100644 models/github_actions/github_actions__current_workflows.sql rename models/github_actions/{github_actions__wf_in_progress.yml => github_actions__current_workflows.yml} (66%) delete mode 100644 models/github_actions/github_actions__wf_in_progress.sql create mode 100644 models/github_actions/github_actions__workflow_history.sql diff --git a/data/github_actions__runtime_detection.csv b/data/github_actions__runtime_detection.csv new file mode 100644 index 00000000..ae095e17 --- /dev/null +++ b/data/github_actions__runtime_detection.csv @@ -0,0 +1,8 @@ +workflow_name +dbt_run_abi_refresh +dbt_run_dev_refresh +dbt_run_operation_reorg +dbt_run_scheduled_curated +dbt_run_scheduled_non_realtime +dbt_run_streamline_chainhead +dbt_run_streamline_decoder \ No newline at end of file diff --git a/models/github_actions/github_actions__current_workflows.sql b/models/github_actions/github_actions__current_workflows.sql new file mode 100644 index 00000000..f63c4d97 --- /dev/null +++ b/models/github_actions/github_actions__current_workflows.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +SELECT + NAME, + status, + created_at, + updated_at, + run_started_at, + run_attempt, + run_number, + TIMESTAMPDIFF(seconds, run_started_at, SYSDATE()) / 60 AS run_minutes, + id, + workflow_id, + html_url +FROM + TABLE( + github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'status' :'in_progress' } + ) + ) +WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }} + ) diff --git a/models/github_actions/github_actions__wf_in_progress.yml b/models/github_actions/github_actions__current_workflows.yml similarity index 66% rename from models/github_actions/github_actions__wf_in_progress.yml rename to models/github_actions/github_actions__current_workflows.yml index 8fd8e7f0..ed95f8b3 100644 --- a/models/github_actions/github_actions__wf_in_progress.yml +++ b/models/github_actions/github_actions__current_workflows.yml @@ -1,8 +1,8 @@ version: 2 models: - - name: github_actions__wf_in_progress + - name: github_actions__current_workflows columns: - - name: DELTA_MINUTES + - name: RUN_MINUTES tests: - dbt_expectations.expect_column_values_to_be_between: max_value: 59 \ No newline at end of file diff --git a/models/github_actions/github_actions__wf_in_progress.sql b/models/github_actions/github_actions__wf_in_progress.sql deleted file mode 100644 index 6a0869a3..00000000 --- a/models/github_actions/github_actions__wf_in_progress.sql +++ /dev/null @@ -1,50 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['gha_workflows'] -) }} - -SELECT - id, - NAME, - node_id, - check_suite_id, - check_suite_node_id, - head_branch, - head_sha, - run_number, - event, - display_title, - status, - conclusion, - workflow_id, - url, - html_url, - pull_requests, - created_at, - updated_at, - actor, - run_attempt, - run_started_at, - triggering_actor, - jobs_url, - logs_url, - check_suite_url, - artifacts_url, - cancel_url, - rerun_url, - workflow_url, - head_commit, - repository, - head_repository, - TIMESTAMPDIFF(seconds, run_started_at, SYSDATE()) / 60 AS delta_minutes -FROM - TABLE( - github_actions.tf_runs( - 'FlipsideCrypto', - 'bsc-models',{ 'status' :'in_progress' } - ) - ) -WHERE - NAME NOT ILIKE '%history%' - AND NAME NOT ILIKE '%test%' - AND NAME NOT ILIKE '%heal%' diff --git a/models/github_actions/github_actions__workflow_history.sql b/models/github_actions/github_actions__workflow_history.sql new file mode 100644 index 00000000..b1178442 --- /dev/null +++ b/models/github_actions/github_actions__workflow_history.sql @@ -0,0 +1,132 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'workflow_history_id', + tags = ['gha_tasks'], + full_refresh = false +) }} + +WITH + +{% if is_incremental() %} +-- get latest +workflow_runs AS ( + + SELECT + id, + NAME, + node_id, + check_suite_id, + check_suite_node_id, + head_branch, + head_sha, + run_number, + event, + display_title, + status, + conclusion, + workflow_id, + url, + html_url, + pull_requests, + created_at, + updated_at, + actor, + run_attempt, + run_started_at, + triggering_actor, + jobs_url, + logs_url, + check_suite_url, + artifacts_url, + cancel_url, + rerun_url, + workflow_url, + head_commit, + repository, + head_repository + FROM + TABLE( + github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'per_page' :'100', + 'page': '1' } + ) + ) + WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }} + ) + AND conclusion IS NOT NULL +) +{% else %} + -- get history, last 100 pages +workflow_runs AS ({% for item in range(100) %} + ( + SELECT + id, + NAME, + node_id, + check_suite_id, + check_suite_node_id, + head_branch, + head_sha, + run_number, + event, + display_title, + status, + conclusion, + workflow_id, + url, + html_url, + pull_requests, + created_at, + updated_at, + actor, + run_attempt, + run_started_at, + triggering_actor, + jobs_url, + logs_url, + check_suite_url, + artifacts_url, + cancel_url, + rerun_url, + workflow_url, + head_commit, + repository, + head_repository + FROM + TABLE(github_actions.tf_runs( + 'FlipsideCrypto', + 'bsc-models',{ 'per_page' :'100', + 'page': '{{ item }}' } + ) + ) + WHERE + NAME IN ( + SELECT + workflow_name + FROM + {{ ref('github_actions__runtime_detection') }}) + AND conclusion IS NOT NULL) + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %}) +{% endif %} + +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['id'] + ) }} AS workflow_history_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + workflow_runs qualify (ROW_NUMBER() over (PARTITION BY id +ORDER BY + created_at DESC)) = 1 From ecad487c9a99a32db90ec34d908e914b983a61b7 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 16 May 2024 11:27:00 -0600 Subject: [PATCH 3/3] run id --- .../github_actions/github_actions__workflow_history.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/models/github_actions/github_actions__workflow_history.sql b/models/github_actions/github_actions__workflow_history.sql index b1178442..5781d5f5 100644 --- a/models/github_actions/github_actions__workflow_history.sql +++ b/models/github_actions/github_actions__workflow_history.sql @@ -13,7 +13,7 @@ WITH workflow_runs AS ( SELECT - id, + id AS run_id, NAME, node_id, check_suite_id, @@ -67,7 +67,7 @@ workflow_runs AS ( workflow_runs AS ({% for item in range(100) %} ( SELECT - id, + id AS run_id, NAME, node_id, check_suite_id, @@ -122,11 +122,11 @@ workflow_runs AS ({% for item in range(100) %} SELECT *, {{ dbt_utils.generate_surrogate_key( - ['id'] + ['run_id'] ) }} AS workflow_history_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp FROM - workflow_runs qualify (ROW_NUMBER() over (PARTITION BY id + workflow_runs qualify (ROW_NUMBER() over (PARTITION BY run_id ORDER BY created_at DESC)) = 1