traces2 (#168)

* traces2

* spacing

.github/workflows/dbt_run_temp_traces2.yml

@@ -0,0 +1,47 @@
+name: dbt_run_temp_traces2
+run-name: dbt_run_temp_traces2
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs “At minute 11 past every hour.” (see https://crontab.guru)
+    - cron: '11 * * * *'
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_2xl
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "gnosis_models,tag:traces_reload"

macros/streamline/models.sql

@@ -57,7 +57,7 @@
             )
 {% endmacro %}
 
-{% macro streamline_external_table_FR_query(
+{% macro streamline_external_table_fr_query(
         model,
         partition_function,
         partition_name,

models/silver/core/silver__blocks.sql

@@ -83,7 +83,7 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_blocks') }}
+    {{ ref('bronze__streamline_fr_blocks') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY block_number

models/silver/core/silver__confirmed_blocks.sql

@@ -32,7 +32,7 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_confirm_blocks') }}
+    {{ ref('bronze__streamline_fr_confirm_blocks') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY block_number

models/silver/core/silver__fact_traces2.sql

@@ -0,0 +1,15 @@
+{{ config (
+    materialized = "incremental",
+    incremental_strategy = 'delete+insert',
+    unique_key = "block_number",
+    incremental_predicates = [fsc_evm.standard_predicate()],
+    cluster_by = "block_timestamp::date",
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
+    tags = ['traces_reload'],
+    full_refresh = false
+) }}
+{{ fsc_evm.gold_traces_v1(
+    full_reload_start_block = 5000000,
+    full_reload_blocks = 1000000,
+    full_reload_mode = true
+) }}

models/silver/core/silver__receipts.sql

@@ -28,7 +28,7 @@ WHERE
     )
     AND IS_OBJECT(DATA)
 {% else %}
-    {{ ref('bronze__streamline_FR_receipts') }}
+    {{ ref('bronze__streamline_fr_receipts') }}
 WHERE
     IS_OBJECT(DATA)
 {% endif %}
@@ -86,4 +86,4 @@ FROM
 WHERE
     tx_hash IS NOT NULL qualify(ROW_NUMBER() over (PARTITION BY block_number, POSITION
 ORDER BY
-    _inserted_timestamp DESC)) = 1
+    _inserted_timestamp DESC)) = 1

models/silver/core/silver__traces.sql

@@ -28,7 +28,7 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_traces') }}
+    {{ ref('bronze__streamline_fr_traces') }}
 WHERE
     _partition_by_block_id <= 5000000
 {% endif %}

models/silver/core/silver__traces2.sql

@@ -8,129 +8,7 @@
     tags = ['non_realtime'],
     full_refresh = false
 ) }}
-
-WITH bronze_traces AS (
-
-    SELECT
-        block_number,
-        _partition_by_block_id AS partition_key,
-        VALUE :array_index :: INT AS tx_position,
-        DATA :result AS full_traces,
-        _inserted_timestamp
-    FROM
-
-{% if is_incremental() %}
-{{ ref('bronze__streamline_traces') }}
-WHERE
-    _inserted_timestamp >= (
-        SELECT
-            MAX(_inserted_timestamp) _inserted_timestamp
-        FROM
-            {{ this }}
-    )
-    AND DATA :result IS NOT NULL
-{% else %}
-    {{ ref('bronze__streamline_FR_traces') }}
-WHERE
-    _partition_by_block_id <= 5000000
-    AND DATA :result IS NOT NULL
-{% endif %}
-
-qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position
-ORDER BY
-    _inserted_timestamp DESC)) = 1
-),
-flatten_traces AS (
-    SELECT
-        block_number,
-        tx_position,
-        partition_key,
-        IFF(
-            path IN (
-                'result',
-                'result.value',
-                'result.type',
-                'result.to',
-                'result.input',
-                'result.gasUsed',
-                'result.gas',
-                'result.from',
-                'result.output',
-                'result.error',
-                'result.revertReason',
-                'gasUsed',
-                'gas',
-                'type',
-                'to',
-                'from',
-                'value',
-                'input',
-                'error',
-                'output',
-                'revertReason'
-            ),
-            'ORIGIN',
-            REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '')
-        ) AS trace_address,
-        _inserted_timestamp,
-        OBJECT_AGG(
-            key,
-            VALUE
-        ) AS trace_json,
-        CASE
-            WHEN trace_address = 'ORIGIN' THEN NULL
-            WHEN POSITION(
-                '_' IN trace_address
-            ) = 0 THEN 'ORIGIN'
-            ELSE REGEXP_REPLACE(
-                trace_address,
-                '_[0-9]+$',
-                '',
-                1,
-                1
-            )
-        END AS parent_trace_address,
-        SPLIT(
-            trace_address,
-            '_'
-        ) AS trace_address_array
-    FROM
-        bronze_traces txs,
-        TABLE(
-            FLATTEN(
-                input => PARSE_JSON(
-                    txs.full_traces
-                ),
-                recursive => TRUE
-            )
-        ) f
-    WHERE
-        f.index IS NULL
-        AND f.key != 'calls'
-        AND f.path != 'result'
-    GROUP BY
-        block_number,
-        tx_position,
-        partition_key,
-        trace_address,
-        _inserted_timestamp
-)
-SELECT
-    block_number,
-    tx_position,
-    trace_address,
-    parent_trace_address,
-    trace_address_array,
-    trace_json,
-    partition_key,
-    _inserted_timestamp,
-    {{ dbt_utils.generate_surrogate_key(
-        ['block_number', 'tx_position', 'trace_address']
-    ) }} AS traces_id,
-    SYSDATE() AS inserted_timestamp,
-    SYSDATE() AS modified_timestamp,
-    '{{ invocation_id }}' AS _invocation_id
-FROM
-    flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id
-ORDER BY
-    _inserted_timestamp DESC)) = 1
+{{ fsc_evm.silver_traces_v1(
+    full_reload_start_block = 5000000,
+    full_reload_blocks = 1000000
+) }}

models/silver/core/silver__transactions.sql

@@ -27,7 +27,7 @@ WHERE
     )
     AND IS_OBJECT(DATA)
 {% else %}
-    {{ ref('bronze__streamline_FR_transactions') }}
+    {{ ref('bronze__streamline_fr_transactions') }}
 WHERE
     IS_OBJECT(DATA)
 {% endif %}

models/streamline/bronze/core/bronze__streamline_FR_blocks.sql → models/streamline/bronze/core/fr/bronze__streamline_fr_blocks.sql

@@ -3,7 +3,7 @@
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}
-{{ streamline_external_table_FR_query(
+{{ streamline_external_table_fr_query(
     model,
     partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
     partition_name = "_partition_by_block_id",

models/streamline/bronze/core/bronze__streamline_FR_confirm_blocks.sql → models/streamline/bronze/core/fr/bronze__streamline_fr_confirm_blocks.sql

@@ -1,7 +1,7 @@
 {{ config (
     materialized = 'view'
 ) }}
-{{ streamline_external_table_FR_query(
+{{ streamline_external_table_fr_query(
     model = "confirm_blocks",
     partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )",
     partition_name = "_partition_by_block_id",

models/streamline/bronze/core/bronze__streamline_FR_transactions.sql → models/streamline/bronze/core/fr/bronze__streamline_fr_receipts.sql

@@ -3,7 +3,7 @@
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}
-{{ streamline_external_table_FR_query(
+{{ streamline_external_table_fr_query(
     model,
     partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )",
     partition_name = "_partition_by_block_id",

models/streamline/bronze/core/bronze__streamline_FR_traces.sql → models/streamline/bronze/core/fr/bronze__streamline_fr_traces.sql

@@ -1,8 +1,7 @@
 {{ config (
     materialized = 'view'
 ) }}
-
-{{ streamline_external_table_FR_query(
+{{ streamline_external_table_fr_query(
     model = "debug_traceBlockByNumber",
     partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )",
     partition_name = "_partition_by_block_id",

models/streamline/bronze/core/bronze__streamline_FR_receipts.sql → models/streamline/bronze/core/fr/bronze__streamline_fr_transactions.sql

@@ -3,7 +3,7 @@
 ) }}
 
 {% set model = this.identifier.split("_") [-1] %}
-{{ streamline_external_table_FR_query(
+{{ streamline_external_table_fr_query(
     model,
     partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )",
     partition_name = "_partition_by_block_id",

models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql

@@ -21,7 +21,7 @@ WHERE
         FROM
             {{ this }})
         {% else %}
-            {{ ref('bronze__streamline_FR_confirm_blocks') }}
+            {{ ref('bronze__streamline_fr_confirm_blocks') }}
         {% endif %}
 
         qualify(ROW_NUMBER() over (PARTITION BY id

models/streamline/silver/core/complete/streamline__complete_debug_traceBlockByNumber.sql

@@ -24,9 +24,9 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_traces') }}
+    {{ ref('bronze__streamline_fr_traces') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY id
 ORDER BY
-    _inserted_timestamp DESC)) = 1
+    _inserted_timestamp DESC)) = 1

models/streamline/silver/core/complete/streamline__complete_qn_getBlockWithReceipts.sql

@@ -24,9 +24,9 @@ WHERE
             {{ this }}
     )
 {% else %}
-    {{ ref('bronze__streamline_FR_blocks') }}
+    {{ ref('bronze__streamline_fr_blocks') }}
 {% endif %}
 
 qualify(ROW_NUMBER() over (PARTITION BY id
 ORDER BY
-    _inserted_timestamp DESC)) = 1
+    _inserted_timestamp DESC)) = 1

package-lock.yml

@@ -9,8 +9,10 @@ packages:
   revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107
 - package: get-select/dbt_snowflake_query_tags
   version: 2.5.0
+- git: https://github.com/FlipsideCrypto/fsc-evm.git
+  revision: ec6adae14ab4060ad4a553fb7f32d7e57693996d
 - package: calogica/dbt_date
   version: 0.7.2
 - git: https://github.com/FlipsideCrypto/livequery-models.git
   revision: b024188be4e9c6bc00ed77797ebdc92d351d620e
-sha1_hash: efa8844f7c3e54f84d660c43f887b9cb084dfd9f
+sha1_hash: 622a679ecf98e6ebf3c904241902ce5328c77e52

packages.yml

@@ -8,4 +8,6 @@ packages:
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
     revision: v1.29.0
   - package: get-select/dbt_snowflake_query_tags
-    version: [">=2.0.0", "<3.0.0"]
+    version: [">=2.0.0", "<3.0.0"]
+  - git: https://github.com/FlipsideCrypto/fsc-evm.git
+    revision: v1.5.0