An 4376/overflow-traces (#226)

* stash * txhash * workflow * prod * drop dev
FlipsideCrypto · Jan 4, 2024 · 28a7897 · 28a7897
1 parent b6b9643
commit 28a7897
Show file tree

Hide file tree

Showing 13 changed files with 526 additions and 21 deletions.
diff --git a/.github/workflows/dbt_run_overflow_models.yml b/.github/workflows/dbt_run_overflow_models.yml
@@ -0,0 +1,44 @@
+name: dbt_run_overflow_models
+run-name: dbt_run_overflow_models
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "optimism_models,tag:overflow"
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -51,6 +51,15 @@ query-comment:
 models:
   +copy_grants: true
   +on_schema_change: "append_new_columns"
+  optimism_models:
+    silver:
+      overflow:
+        silver__overflowed_traces:
+          +snowflake_warehouse: "DBT_SNOWPARK"
+    bronze:
+      overflow:
+        bronze__overflowed_traces:
+          +snowflake_warehouse: "DBT_SNOWPARK"
 
 # In this example config, we tell dbt to build all models in the example/ directory
 # as tables. These settings can be overridden in the individual model files

diff --git a/models/bronze/overflow/bronze__overflowed_traces.sql b/models/bronze/overflow/bronze__overflowed_traces.sql
@@ -0,0 +1,79 @@
+{{ config (
+    materialized = "view"
+) }}
+
+{% for item in range(
+        1,
+        11
+    ) %}
+
+    SELECT
+        o.file_name,
+        f.block_number,
+        f.index_vals,
+        f.path,
+        f.key,
+        f.value_
+    FROM
+        (
+            SELECT
+                file_name,
+                file_url,
+                index_cols,
+                [overflowed_block, overflowed_tx] AS index_vals
+            FROM
+                (
+                    SELECT
+                        block_number,
+                        POSITION,
+                        file_name,
+                        file_url,
+                        index_cols,
+                        VALUE [0] AS overflowed_block,
+                        VALUE [1] AS overflowed_tx,
+                        block_number = overflowed_block
+                        AND POSITION = overflowed_tx AS missing
+                    FROM
+                        (
+                            SELECT
+                                block_number,
+                                POSITION,
+                                file_name,
+                                file_url,
+                                index_cols,
+                                utils.udf_detect_overflowed_responses(
+                                    file_url,
+                                    index_cols
+                                ) AS index_vals
+                            FROM
+                                {{ ref("bronze__potential_overflowed_traces") }}
+                            WHERE
+                                row_no = {{ item }}
+                        ),
+                        LATERAL FLATTEN (
+                            input => index_vals
+                        )
+                )
+            WHERE
+                missing = TRUE
+        ) o,
+        TABLE(
+            utils.udtf_flatten_overflowed_responses(
+                o.file_url,
+                o.index_cols,
+                [o.index_vals]
+            )
+        ) f
+    WHERE
+        NOT IS_OBJECT(
+            f.value_
+        )
+        AND NOT IS_ARRAY(
+            f.value_
+        )
+        AND NOT IS_NULL_VALUE(
+            f.value_
+        ) {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+    {% endfor %}
diff --git a/models/bronze/overflow/bronze__potential_overflowed_traces.sql b/models/bronze/overflow/bronze__potential_overflowed_traces.sql
@@ -0,0 +1,67 @@
+{{ config (
+    materialized = "view"
+) }}
+
+WITH impacted_blocks AS (
+
+    SELECT
+        VALUE :: INT AS block_number
+    FROM
+        (
+            SELECT
+                blocks_impacted_array
+            FROM
+                {{ ref("silver_observability__traces_completeness") }}
+            ORDER BY
+                test_timestamp DESC
+            LIMIT
+                1
+        ), LATERAL FLATTEN (
+            input => blocks_impacted_array
+        )
+),
+all_txs AS (
+    SELECT
+        t.block_number,
+        t.position,
+        t.tx_hash
+    FROM
+        {{ ref("silver__transactions") }}
+        t
+        JOIN impacted_blocks USING (block_number)
+),
+missing_txs AS (
+    SELECT
+        DISTINCT block_number,
+        POSITION,
+        file_name
+    FROM
+        all_txs
+        LEFT JOIN {{ ref("core__fact_traces") }}
+        tr USING (
+            block_number,
+            tx_hash
+        )
+        JOIN {{ ref("streamline__complete_debug_traceBlockByNumber") }} USING (block_number)
+    WHERE
+        tr.tx_hash IS NULL
+)
+SELECT
+    block_number,
+    POSITION,
+    file_name,
+    build_scoped_file_url(
+        @streamline.bronze.external_tables,
+        file_name
+    ) AS file_url,
+    ['block_number', 'array_index'] AS index_cols,
+    ROW_NUMBER() over (
+        ORDER BY
+            block_number ASC,
+            POSITION ASC
+    ) AS row_no
+FROM
+    missing_txs
+ORDER BY
+    block_number ASC,
+    POSITION ASC
diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql
@@ -44,3 +44,47 @@ SELECT
     eth_value_precise
 FROM
     {{ ref('silver__traces') }}
+UNION ALL
+SELECT
+    tx_hash,
+    block_number,
+    block_timestamp,
+    from_address,
+    to_address,
+    eth_value AS VALUE,
+    eth_value_precise_raw AS value_precise_raw,
+    eth_value_precise AS value_precise,
+    gas,
+    gas_used,
+    input,
+    output,
+    TYPE,
+    identifier,
+    DATA,
+    tx_status,
+    sub_traces,
+    trace_status,
+    error_reason,
+    trace_index,
+    COALESCE (
+        traces_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'trace_index']
+        ) }}
+    ) AS fact_traces_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp,
+    eth_value,
+    eth_value_precise_raw,
+    eth_value_precise
+FROM
+    {{ source(
+        'optimism_silver',
+        'overflowed_traces'
+    ) }}
diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql
@@ -81,7 +81,7 @@ broken_blocks AS (
     FROM
         {{ ref("silver__transactions") }}
         tx
-        LEFT JOIN {{ ref("silver__traces") }}
+        LEFT JOIN {{ ref("core__fact_traces") }}
         tr USING (
             block_number,
             tx_hash

diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql
@@ -6,11 +6,11 @@
 SELECT
     *
 FROM
-    {{ ref('silver__traces') }}
+    {{ ref('core__fact_traces') }}
 WHERE
     block_number NOT IN (
-    SELECT
-        block_number
-    FROM
-        {{ ref('silver_observability__excluded_receipt_blocks') }}
-    )
+        SELECT
+            block_number
+        FROM
+            {{ ref('silver_observability__excluded_receipt_blocks') }}
+    )
diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml
@@ -4,8 +4,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - BLOCK_NUMBER
-            - TX_POSITION
+            - TX_HASH
             - TRACE_INDEX
     columns:
       - name: BLOCK_NUMBER
@@ -17,8 +16,7 @@ models:
                 - FLOAT    
       - name: BLOCK_TIMESTAMP
         tests:
-          - not_null:
-              where: NOT IS_PENDING
+          - not_null
           - dbt_expectations.expect_row_values_to_have_recent_data:
               datepart: day
               interval: 1
@@ -28,8 +26,7 @@ models:
                 - TIMESTAMP_NTZ
       - name: TX_HASH
         tests:
-          - not_null:
-              where: NOT IS_PENDING
+          - not_null
           - dbt_expectations.expect_column_values_to_match_regex:
               regex: 0[xX][0-9a-fA-F]+
       - name: FROM_ADDRESS

diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql
@@ -13,7 +13,7 @@ WITH last_3_days AS (
 SELECT
     *
 FROM
-    {{ ref('silver__traces') }}
+    {{ ref('core__fact_traces') }}
 WHERE
     block_number >= (
         SELECT

diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml
@@ -4,8 +4,7 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns:
-            - BLOCK_NUMBER
-            - TX_POSITION
+            - TX_HASH
             - TRACE_INDEX
     columns:
       - name: BLOCK_NUMBER
@@ -17,8 +16,7 @@ models:
                 - FLOAT    
       - name: BLOCK_TIMESTAMP
         tests:
-          - not_null:
-              where: NOT IS_PENDING
+          - not_null
           - dbt_expectations.expect_row_values_to_have_recent_data:
               datepart: day
               interval: 1
@@ -28,8 +26,7 @@ models:
                 - TIMESTAMP_NTZ
       - name: TX_HASH
         tests:
-          - not_null:
-              where: NOT IS_PENDING
+          - not_null
           - dbt_expectations.expect_column_values_to_match_regex:
               regex: 0[xX][0-9a-fA-F]+
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,7 +13,7 @@ WITH last_3_days AS ( @@
     SELECT
         *
     FROM
-        {{ ref('silver__traces') }}
+        {{ ref('core__fact_traces') }}
     WHERE
         block_number >= (
             SELECT
@@ Expand Down @@