diff --git a/.github/workflows/dbt_run_overflowed_traces2.yml b/.github/workflows/dbt_run_overflowed_traces.yml similarity index 80% rename from .github/workflows/dbt_run_overflowed_traces2.yml rename to .github/workflows/dbt_run_overflowed_traces.yml index a71c2400..66cd39fc 100644 --- a/.github/workflows/dbt_run_overflowed_traces2.yml +++ b/.github/workflows/dbt_run_overflowed_traces.yml @@ -1,11 +1,11 @@ -name: dbt_run_overflowed_traces2 -run-name: dbt_run_overflowed_traces2 +name: dbt_run_overflowed_traces +run-name: dbt_run_overflowed_traces on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -21,12 +21,10 @@ env: concurrency: group: ${{ github.workflow }} - - jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -43,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "base_models,tag:overflowed_traces2" --vars '{"OVERFLOWED_TRACES":True}' \ No newline at end of file + dbt run -m "base_models,tag:overflowed_traces" --vars '{"OVERFLOWED_TRACES":True}' diff --git a/.github/workflows/dbt_run_traces_fix.yml b/.github/workflows/dbt_run_traces_fix.yml deleted file mode 100644 index 441e10fe..00000000 --- a/.github/workflows/dbt_run_traces_fix.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_traces_fix -run-name: dbt_run_traces_fix - -on: - workflow_dispatch: - schedule: - # every 30 minutes (see https://crontab.guru) - - cron: "*/30 * * * *" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: run traces fix model - run: | - dbt run -m "base_models,tag:traces_fix" diff --git a/models/bronze/overflow/bronze__overflowed_traces2.sql b/models/bronze/overflow/bronze__overflowed_traces.sql similarity index 97% rename from models/bronze/overflow/bronze__overflowed_traces2.sql rename to models/bronze/overflow/bronze__overflowed_traces.sql index 2be73b9e..d10f9817 100644 --- a/models/bronze/overflow/bronze__overflowed_traces2.sql +++ b/models/bronze/overflow/bronze__overflowed_traces.sql @@ -1,6 +1,6 @@ {{ config ( materialized = "view", - tags = ['overflowed_traces2'] + tags = ['overflowed_traces'] ) }} {% for item in range( @@ -47,7 +47,7 @@ index_cols ) AS index_vals FROM - {{ ref("bronze__potential_overflowed_traces2") }} + {{ ref("bronze__potential_overflowed_traces") }} WHERE row_no = {{ item }} ), diff --git a/models/bronze/overflow/bronze__potential_overflowed_traces2.sql b/models/bronze/overflow/bronze__potential_overflowed_traces.sql similarity index 90% rename from models/bronze/overflow/bronze__potential_overflowed_traces2.sql rename to models/bronze/overflow/bronze__potential_overflowed_traces.sql index 70eed7e9..ab113428 100644 --- a/models/bronze/overflow/bronze__potential_overflowed_traces2.sql +++ b/models/bronze/overflow/bronze__potential_overflowed_traces.sql @@ -1,6 +1,6 @@ {{ config ( materialized = "view", - tags = ['overflowed_traces2'] + tags = ['overflowed_traces'] ) }} WITH impacted_blocks AS ( @@ -38,7 +38,10 @@ missing_txs AS ( file_name FROM all_txs txs - LEFT JOIN {{ ref("silver__traces2") }} + LEFT JOIN {{ source( + 'base_gold', + 'fact_traces' + ) }} tr2 USING ( block_number, tx_position @@ -46,7 +49,7 @@ missing_txs AS ( JOIN {{ ref("streamline__traces_complete") }} USING (block_number) LEFT JOIN {{ source( 'base_silver', - 'overflowed_traces2' + 'overflowed_traces' ) }} ot USING ( block_number, diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index f87f6d8c..944e4ffc 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -1,43 +1,526 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true } +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + incremental_predicates = [fsc_evm.standard_predicate()], + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core','non_realtime'], + full_refresh = false ) }} +WITH silver_traces AS ( + + SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'regular' AS source + FROM + {{ ref('silver__traces') }} + WHERE + 1 = 1 + +{% if is_incremental() and not full_reload_mode %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) {% elif is_incremental() and full_reload_mode %} +AND block_number BETWEEN ( + SELECT + MAX( + block_number + ) + FROM + {{ this }} +) +AND ( + SELECT + MAX( + block_number + ) + 1000000 + FROM + {{ this }} +) +{% else %} + AND block_number <= 3000000 +{% endif %} +UNION ALL +SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'overflow' AS source +FROM + {{ ref('silver__overflowed_traces') }} +WHERE + 1 = 1 + +{% if is_incremental() and not full_reload_mode %} +AND modified_timestamp > ( + SELECT + DATEADD('hour', -2, MAX(modified_timestamp)) + FROM + {{ this }}) {% elif is_incremental() and full_reload_mode %} + AND block_number BETWEEN ( + SELECT + MAX( + block_number + ) + FROM + {{ this }} + ) + AND ( + SELECT + MAX( + block_number + ) + 1000000 + FROM + {{ this }} + ) + {% else %} + AND block_number <= 3000000 + {% endif %} +), +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + silver_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), +trace_index_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS number_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + silver_traces, + LATERAL FLATTEN ( + input => trace_address_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address +), +trace_index_sub_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + number_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + number_array ASC + ) - 1 AS trace_index, + b.trace_json, + b.traces_id, + b.source + FROM + silver_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN trace_index_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), +errored_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + trace_json + FROM + trace_index_sub_traces + WHERE + trace_json :error :: STRING IS NOT NULL +), +error_logic AS ( + SELECT + b0.block_number, + b0.tx_position, + b0.trace_address, + b0.trace_json :error :: STRING AS error, + b1.trace_json :error :: STRING AS any_error, + b2.trace_json :error :: STRING AS origin_error + FROM + trace_index_sub_traces b0 + LEFT JOIN errored_traces b1 + ON b0.block_number = b1.block_number + AND b0.tx_position = b1.tx_position + AND b0.trace_address RLIKE CONCAT( + '^', + b1.trace_address, + '(_[0-9]+)*$' + ) + LEFT JOIN errored_traces b2 + ON b0.block_number = b2.block_number + AND b0.tx_position = b2.tx_position + AND b2.trace_address = 'ORIGIN' +), +aggregated_errors AS ( + SELECT + block_number, + tx_position, + trace_address, + error, + IFF(MAX(any_error) IS NULL + AND error IS NULL + AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded + FROM + error_logic + GROUP BY + block_number, + tx_position, + trace_address, + error, + origin_error), + json_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + sub_traces, + number_array, + trace_index, + trace_json AS DATA, + trace_succeeded, + trace_json :error :: STRING AS error_reason, + trace_json :revertReason :: STRING AS revert_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + trace_json :value :: STRING, + '0x0' + ) AS value_hex, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + IFF( + trace_succeeded, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + traces_id + FROM + trace_index_sub_traces + JOIN aggregated_errors USING ( + block_number, + tx_position, + trace_address + ) + ), + incremental_traces AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.origin_function_signature, + t.from_address AS origin_from_address, + t.to_address AS origin_to_address, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address AS from_address, + f.to_address AS to_address, + f.value_hex, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + f.revert_reason, + f.trace_status, + f.data, + f.traces_id, + f.trace_succeeded, + f.trace_address, + IFF( + t.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + json_traces f + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON f.tx_position = t.position + AND f.block_number = t.block_number + +{% if is_incremental() and not full_reload_mode %} +AND t.modified_timestamp >= ( + SELECT + DATEADD('hour', -24, MAX(modified_timestamp)) + FROM + {{ this }}) + {% endif %} +) + +{% if is_incremental() %}, +overflow_blocks AS ( + SELECT + DISTINCT block_number + FROM + silver_traces + WHERE + source = 'overflow' +), +heal_missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp, + txs.origin_function_signature, + txs.from_address AS origin_from_address, + txs.to_address AS origin_to_address, + txs.tx_status, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_hex, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.revert_reason, + t.trace_status, + t.data, + t.fact_traces_id AS traces_id, + t.trace_succeeded, + t.trace_address, + IFF( + txs.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + {{ this }} + t + JOIN {{ ref('silver__transactions') }} + txs + ON t.tx_position = txs.position + AND t.block_number = txs.block_number + WHERE + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL +) +{% endif %}, +all_traces AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded + FROM + incremental_traces + +{% if is_incremental() %} +UNION ALL SELECT - tx_hash, block_number, + tx_hash, block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, from_address, to_address, - eth_value AS VALUE, - eth_value_precise_raw AS value_precise_raw, - eth_value_precise AS value_precise, + value_hex, + value_precise_raw, + value_precise, + VALUE, gas, gas_used, input, output, TYPE, identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + heal_missing_data +UNION ALL +SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, sub_traces, - trace_status, error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + {{ this }} + JOIN overflow_blocks USING (block_number) +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, trace_index, - COALESCE ( - traces_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} - ) AS fact_traces_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + from_address, + to_address, + input, + output, + TYPE, + trace_address, + sub_traces, + VALUE, + value_precise_raw, + value_precise, + value_hex, + gas, + gas_used, + origin_from_address, + origin_to_address, + origin_function_signature, + trace_succeeded, + error_reason, + revert_reason, + tx_succeeded, + identifier, + DATA, + tx_status, + trace_status, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS fact_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp FROM - {{ ref('silver__traces') }} + all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index +ORDER BY + modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml index cb7b08f4..53d2df9b 100644 --- a/models/gold/core/core__fact_traces.yml +++ b/models/gold/core/core__fact_traces.yml @@ -1,56 +1,69 @@ version: 2 models: - name: core__fact_traces - description: '{{ doc("base_traces_table_doc") }}' + description: '{{ doc("base_traces_table_doc") }}' columns: - name: BLOCK_NUMBER - description: '{{ doc("base_traces_block_no") }}' + description: '{{ doc("base_traces_block_no") }}' - name: BLOCK_TIMESTAMP description: '{{ doc("base_traces_blocktime") }}' - name: TX_HASH - description: '{{ doc("base_traces_tx_hash") }}' + description: '{{ doc("base_traces_tx_hash") }}' + - name: TX_POSITION + description: The position of the transaction within the block. + - name: TRACE_INDEX + description: The index of the trace within the transaction. - name: FROM_ADDRESS description: '{{ doc("base_traces_from") }}' - name: TO_ADDRESS description: '{{ doc("base_traces_to") }}' - - name: VALUE - description: '{{ doc("base_traces_value") }}' - - name: VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' - - name: VALUE_PRECISE - description: '{{ doc("precise_amount_adjusted") }}' - - name: GAS - description: '{{ doc("base_traces_gas") }}' - - name: GAS_USED - description: '{{ doc("base_traces_gas_used") }}' - name: INPUT description: '{{ doc("base_traces_input") }}' - name: OUTPUT description: '{{ doc("base_traces_output") }}' - name: TYPE description: '{{ doc("base_traces_type") }}' + - name: TRACE_ADDRESS + description: This field represents the position of the trace within the transaction. + - name: SUB_TRACES + description: '{{ doc("base_traces_sub") }}' - name: IDENTIFIER description: '{{ doc("base_traces_identifier") }}' - name: DATA description: '{{ doc("base_traces_call_data") }}' - - name: TX_STATUS - description: '{{ doc("base_tx_status") }}' - - name: SUB_TRACES - description: '{{ doc("base_traces_sub") }}' + + - name: VALUE + description: '{{ doc("base_traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: GAS + description: '{{ doc("base_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("base_traces_gas_used") }}' + - name: ORIGIN_FROM_ADDRESS + description: The from address at the transaction level. + - name: ORIGIN_TO_ADDRESS + description: The to address at the transaction level. + - name: ORIGIN_FUNCTION_SIGNATURE + description: The function signature at the transaction level. - name: TRACE_STATUS description: The status of the trace, either `SUCCESS` or `FAIL` + - name: TRACE_SUCCEEDED + description: Whether the trace succeeded or failed - name: ERROR_REASON description: The reason for the trace failure, if any. - - name: TRACE_INDEX - description: The index of the trace within the transaction. + - name: REVERT_REASON + description: The reason for the trace revert, if any. + - name: TX_STATUS + description: '{{ doc("base_tx_status") }}' + - name: TX_SUCCEEDED + description: Whether the transaction succeeded or failed - name: FACT_TRACES_ID - description: '{{ doc("pk") }}' + description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - - - - \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_full.sql b/models/gold/core/tests/traces/test_gold__fact_traces_full.sql new file mode 100644 index 00000000..ced1fba5 --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = "view", + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('core__fact_traces') }} diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_full.yml b/models/gold/core/tests/traces/test_gold__fact_traces_full.yml new file mode 100644 index 00000000..6677117b --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_full + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql b/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql new file mode 100644 index 00000000..4bed00ff --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_recent.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = "view", + tags = ['recent_test'] +) }} + +SELECT + * +FROM + {{ ref('core__fact_traces') }} +WHERE + block_number > ( + SELECT + block_number + FROM + {{ ref('_block_lookback') }} + ) diff --git a/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml b/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml new file mode 100644 index 00000000..4206f508 --- /dev/null +++ b/models/gold/core/tests/traces/test_gold__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_recent + description: "This is a view used to test the last three days of fact traces." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql index a4ca260f..53bbcb1d 100644 --- a/models/silver/_observability/silver_observability__traces_completeness.sql +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -81,7 +81,10 @@ broken_blocks AS ( FROM {{ ref("silver__transactions") }} tx - LEFT JOIN {{ ref("silver__traces") }} + LEFT JOIN {{ source( + "base_gold", + "fact_traces" + ) }} tr USING ( block_number, tx_hash diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql index bb35fa32..3cceeaf1 100644 --- a/models/silver/abis/silver__proxies.sql +++ b/models/silver/abis/silver__proxies.sql @@ -11,9 +11,9 @@ WITH base AS ( from_address, to_address, MIN(block_number) AS start_block, - MAX(_inserted_timestamp) AS _inserted_timestamp + MAX(modified_timestamp) AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE = 'DELEGATECALL' AND trace_status = 'SUCCESS' @@ -21,7 +21,7 @@ WITH base AS ( AND from_address != to_address -- exclude self-calls {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '24 hours' FROM @@ -99,5 +99,6 @@ FROM FINAL f JOIN {{ ref('silver__created_contracts') }} C ON f.contract_address = C.created_contract_address - JOIN {{ ref('silver__created_contracts') }} p - ON f.proxy_address = p.created_contract_address \ No newline at end of file + JOIN {{ ref('silver__created_contracts') }} + p + ON f.proxy_address = p.created_contract_address diff --git a/models/silver/abis/silver__user_verified_abis.sql b/models/silver/abis/silver__user_verified_abis.sql index 21f4109f..fcab5b01 100644 --- a/models/silver/abis/silver__user_verified_abis.sql +++ b/models/silver/abis/silver__user_verified_abis.sql @@ -473,7 +473,7 @@ valid_traces AS ( SELECT base_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} JOIN function_mapping ON function_sig = LEFT( input, diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index 3ec66a4b..bbc1f300 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -13,7 +13,7 @@ SELECT to_address AS created_contract_address, from_address AS creator_address, input AS created_contract_input, - _inserted_timestamp, + modified_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['to_address'] ) }} AS created_contracts_id, @@ -21,7 +21,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE ILIKE 'create%' AND to_address IS NOT NULL @@ -63,4 +63,4 @@ FROM {% if is_incremental() %} WHERE 1 = 2 -{% endif %} \ No newline at end of file +{% endif %} diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql deleted file mode 100644 index 9d5dff36..00000000 --- a/models/silver/core/silver__fact_traces2.sql +++ /dev/null @@ -1,521 +0,0 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - incremental_predicates = [fsc_evm.standard_predicate()], - cluster_by = "block_timestamp::date", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core','non_realtime'], - full_refresh = false -) }} - - -WITH silver_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - traces_id, - 'regular' AS source - FROM - {{ ref('silver__traces2') }} - WHERE - 1 = 1 - -{% if is_incremental() and not full_reload_mode %} -AND modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) {% elif is_incremental() and full_reload_mode %} -AND block_number BETWEEN ( - SELECT - MAX( - block_number - ) - FROM - {{ this }} -) -AND ( - SELECT - MAX( - block_number - ) + 1000000 - FROM - {{ this }} -) -{% else %} - AND block_number <= 3000000 -{% endif %} - -UNION ALL -SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - traces_id, - 'overflow' AS source -FROM - {{ ref('silver__overflowed_traces2') }} - -WHERE - 1 = 1 - -{% if is_incremental() and not full_reload_mode %} -AND modified_timestamp > ( - SELECT - DATEADD('hour', -2, MAX(modified_timestamp)) - FROM - {{ this }}) {% elif is_incremental() and full_reload_mode %} - AND block_number BETWEEN ( - SELECT - MAX( - block_number - ) - FROM - {{ this }} - ) - AND ( - SELECT - MAX( - block_number - ) + 1000000 - FROM - {{ this }} - ) - {% else %} - AND block_number <= 3000000 - {% endif %} -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - silver_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -trace_index_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS number_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - silver_traces, - LATERAL FLATTEN ( - input => trace_address_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -trace_index_sub_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - number_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, b.tx_position ORDER BY number_array ASC - ) - 1 AS trace_index, - b.trace_json, - b.traces_id, - b.source - FROM - silver_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN trace_index_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -errored_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - trace_json - FROM - trace_index_sub_traces - WHERE - trace_json :error :: STRING IS NOT NULL -), -error_logic AS ( - SELECT - b0.block_number, - b0.tx_position, - b0.trace_address, - b0.trace_json :error :: STRING AS error, - b1.trace_json :error :: STRING AS any_error, - b2.trace_json :error :: STRING AS origin_error - FROM - trace_index_sub_traces b0 - LEFT JOIN errored_traces b1 - ON b0.block_number = b1.block_number - AND b0.tx_position = b1.tx_position - AND b0.trace_address RLIKE CONCAT('^', b1.trace_address, '(_[0-9]+)*$') - LEFT JOIN errored_traces b2 - ON b0.block_number = b2.block_number - AND b0.tx_position = b2.tx_position - AND b2.trace_address = 'ORIGIN' -), -aggregated_errors AS ( - SELECT - block_number, - tx_position, - trace_address, - error, - IFF(MAX(any_error) IS NULL - AND error IS NULL - AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded - FROM - error_logic - GROUP BY - block_number, - tx_position, - trace_address, - error, - origin_error - ), - json_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - sub_traces, - number_array, - trace_index, - trace_json AS DATA, - trace_succeeded, - trace_json :error :: STRING AS error_reason, - trace_json :revertReason :: STRING AS revert_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - trace_json :value :: STRING, - '0x0' - ) AS value_hex, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS value_precise_raw, - utils.udf_decimal_adjust( - value_precise_raw, - 18 - ) AS value_precise, - value_precise :: FLOAT AS VALUE, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - IFF( - trace_succeeded, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - traces_id - FROM - trace_index_sub_traces - JOIN aggregated_errors USING ( - block_number, - tx_position, - trace_address - ) - ), - incremental_traces AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.origin_function_signature, - t.from_address AS origin_from_address, - t.to_address AS origin_to_address, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address AS from_address, - f.to_address AS to_address, - f.value_hex, - f.value_precise_raw, - f.value_precise, - f.value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - f.revert_reason, - f.trace_status, - f.data, - f.traces_id, - f.trace_succeeded, - f.trace_address, - IFF( - t.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - json_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() and not full_reload_mode %} -AND t.modified_timestamp >= ( - SELECT - DATEADD('hour', -24, MAX(modified_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -overflow_blocks AS ( - SELECT - DISTINCT block_number - FROM - silver_traces - WHERE - source = 'overflow' -), -heal_missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.origin_function_signature, - txs.from_address AS origin_from_address, - txs.to_address AS origin_to_address, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.value_hex, - t.value_precise_raw, - t.value_precise, - t.value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.revert_reason, - t.trace_status, - t.data, - t.fact_traces_id AS traces_id, - t.trace_succeeded, - t.trace_address, - IFF( - txs.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - {{ this }} - t - JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL -) -{% endif %}, -all_traces AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded - FROM - incremental_traces - -{% if is_incremental() %} -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - heal_missing_data -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - {{ this }} - JOIN overflow_blocks USING (block_number) -{% endif %} -) -SELECT - block_number, - block_timestamp, - tx_hash, - tx_position, - trace_index, - from_address, - to_address, - input, - output, - TYPE, - trace_address, - sub_traces, - VALUE, - value_precise_raw, - value_precise, - value_hex, - gas, - gas_used, - origin_from_address, - origin_to_address, - origin_function_signature, - trace_succeeded, - error_reason, - revert_reason, - tx_succeeded, - identifier, - DATA, - tx_status, - trace_status, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} AS fact_traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index -ORDER BY - modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 \ No newline at end of file diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql index 57c83126..63c9c790 100644 --- a/models/silver/core/silver__native_transfers.sql +++ b/models/silver/core/silver__native_transfers.sql @@ -16,17 +16,26 @@ WITH eth_base AS ( identifier, from_address, to_address, - eth_value, - _call_id, - _inserted_timestamp, - eth_value_precise_raw, - eth_value_precise, + VALUE AS eth_value, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp, + value_precise_raw AS eth_value_precise_raw, + value_precise AS eth_value_precise, tx_position, trace_index FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE - eth_value > 0 + VALUE > 0 AND tx_status = 'SUCCESS' AND trace_status = 'SUCCESS' AND TYPE NOT IN ( @@ -98,7 +107,8 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM eth_base A - LEFT JOIN {{ ref('silver__complete_token_prices') }} p + LEFT JOIN {{ ref('silver__complete_token_prices') }} + p ON DATE_TRUNC( 'hour', A.block_timestamp diff --git a/models/silver/core/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql index 85b5b2bc..23aefb70 100644 --- a/models/silver/core/silver__relevant_contracts.sql +++ b/models/silver/core/silver__relevant_contracts.sql @@ -31,10 +31,10 @@ function_calls AS ( SELECT to_address AS contract_address, COUNT(*) AS function_call_count, - MAX(_inserted_timestamp) AS max_inserted_timestamp_traces, + MAX(modified_timestamp) AS max_inserted_timestamp_traces, MAX(block_number) AS latest_call_block FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE tx_status = 'SUCCESS' AND trace_status = 'SUCCESS' @@ -43,7 +43,7 @@ function_calls AS ( AND input <> '0x' {% if is_incremental() %} -AND _inserted_timestamp > ( +AND modified_timestamp > ( SELECT MAX(max_inserted_timestamp_traces) FROM diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index a0b7f4b5..1d54d032 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -3,23 +3,31 @@ materialized = "incremental", incremental_strategy = 'delete+insert', unique_key = "block_number", - cluster_by = "block_timestamp::date, _inserted_timestamp::date", + cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", full_refresh = false, tags = ['core','non_realtime'] ) }} - +{# {{ silver_traces_v1( +full_reload_start_block = 2300000, +full_reload_blocks = 1000000, +use_partition_key = TRUE +) }} +#} WITH bronze_traces AS ( SELECT block_number, + partition_key, VALUE :array_index :: INT AS tx_position, DATA :result AS full_traces, _inserted_timestamp FROM -{% if is_incremental() %} -{{ ref('bronze__traces') }} +{% if is_incremental() and not full_reload_mode %} +{{ ref( + 'bronze__traces' +) }} WHERE _inserted_timestamp >= ( SELECT @@ -27,12 +35,29 @@ WHERE FROM {{ this }} ) - AND DATA :result IS NOT NULL + AND DATA :result IS NOT NULL {% elif is_incremental() and full_reload_mode %} + {{ ref( + 'bronze__traces_fr' + ) }} +WHERE + partition_key BETWEEN ( + SELECT + MAX(partition_key) - 100000 + FROM + {{ this }} + ) + AND ( + SELECT + MAX(partition_key) + 1000000 + FROM + {{ this }} + ) {% else %} - {{ ref('bronze__traces_fr') }} + {{ ref( + 'bronze__traces_fr' + ) }} WHERE - _partition_by_block_id <= 2300000 - AND DATA :result IS NOT NULL + partition_key <= 2300000 {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position @@ -43,6 +68,7 @@ flatten_traces AS ( SELECT block_number, tx_position, + partition_key, IFF( path IN ( 'result', @@ -56,6 +82,7 @@ flatten_traces AS ( 'result.output', 'result.error', 'result.revertReason', + 'result.time', 'gasUsed', 'gas', 'type', @@ -65,6 +92,7 @@ flatten_traces AS ( 'input', 'error', 'output', + 'time', 'revertReason' ), 'ORIGIN', @@ -91,7 +119,7 @@ flatten_traces AS ( SPLIT( trace_address, '_' - ) AS str_array + ) AS trace_address_array FROM bronze_traces txs, TABLE( @@ -109,316 +137,26 @@ flatten_traces AS ( GROUP BY block_number, tx_position, + partition_key, trace_address, _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS eth_value_precise_raw, - utils.udf_decimal_adjust( - eth_value_precise_raw, - 18 - ) AS eth_value_precise, - eth_value_precise :: FLOAT AS eth_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces -), -new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.eth_value_precise_raw, - f.eth_value_precise, - f.eth_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() %} -AND t._INSERTED_TIMESTAMP >= ( - SELECT - DATEADD('hour', -24, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.eth_value_precise_raw, - t.eth_value_precise, - t.eth_value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.trace_status, - t.data, - IFF( - txs.tx_hash IS NULL - OR txs.block_timestamp IS NULL - OR txs.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - t._call_id, - GREATEST( - t._inserted_timestamp, - txs._inserted_timestamp - ) AS _inserted_timestamp - FROM - {{ this }} - t - INNER JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.is_pending -) -{% endif %}, -FINAL AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - eth_value_precise_raw, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - missing_data -{% endif %} ) SELECT block_number, - tx_hash, - block_timestamp, - tx_status, tx_position, - trace_index, - from_address, - to_address, - eth_value_precise, - eth_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, _inserted_timestamp, - eth_value_precise_raw, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] + ['block_number', 'tx_position', 'trace_address'] ) }} AS traces_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index + flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id ORDER BY - _inserted_timestamp DESC, is_pending ASC)) = 1 + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql deleted file mode 100644 index ab951f99..00000000 --- a/models/silver/core/silver__traces2.sql +++ /dev/null @@ -1,15 +0,0 @@ --- depends_on: {{ ref('bronze__traces') }} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - full_refresh = false, - tags = ['core','non_realtime'] -) }} -{{ silver_traces_v1( - full_reload_start_block = 2300000, - full_reload_blocks = 1000000, - use_partition_key = true -) }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql index e1226946..45a66a21 100644 --- a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__blocks') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql index adbbf1bb..d7655c7f 100644 --- a/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql +++ b/models/silver/core/tests/confirmed_blocks/test_silver__confirmed_blocks_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__confirmed_blocks') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/decoded_logs/test_silver__decoded_logs_recent.sql b/models/silver/core/tests/decoded_logs/test_silver__decoded_logs_recent.sql index 3587de31..224f4dc3 100644 --- a/models/silver/core/tests/decoded_logs/test_silver__decoded_logs_recent.sql +++ b/models/silver/core/tests/decoded_logs/test_silver__decoded_logs_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__decoded_logs') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql index 03400cc3..879aa9b5 100644 --- a/models/silver/core/tests/event_logs/test_silver__logs_recent.sql +++ b/models/silver/core/tests/event_logs/test_silver__logs_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__logs') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql index f83874de..97bd45a5 100644 --- a/models/silver/core/tests/receipts/test_silver__receipts_recent.sql +++ b/models/silver/core/tests/receipts/test_silver__receipts_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__receipts') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/traces/test_silver__traces_full.yml b/models/silver/core/tests/traces/test_silver__traces_full.yml index 639f77bd..f21a7e1d 100644 --- a/models/silver/core/tests/traces/test_silver__traces_full.yml +++ b/models/silver/core/tests/traces/test_silver__traces_full.yml @@ -1,59 +1,52 @@ version: 2 models: - name: test_silver__traces_full + description: "This is a view used to test all of the silver traces model." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_LTZ - - TIMESTAMP_NTZ - - name: TX_HASH - tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: FROM_ADDRESS + - not_null + - name: TX_POSITION tests: - - not_null: - where: TYPE <> 'SELFDESTRUCT' - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - - name: TO_ADDRESS + - not_null + - name: TRACE_ADDRESS tests: - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - where: TO_ADDRESS IS NOT NULL - - name: IDENTIFIER + - not_null + - name: TRACE_JSON tests: - not_null - - name: ETH_VALUE + - name: _INSERTED_TIMESTAMP tests: - not_null - - name: GAS + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP tests: - not_null - - name: GAS_USED + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: MODIFIED_TIMESTAMP tests: - not_null - - + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql index 1dc3e769..493b8f2c 100644 --- a/models/silver/core/tests/traces/test_silver__traces_recent.sql +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__traces') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.yml b/models/silver/core/tests/traces/test_silver__traces_recent.yml index 51db5b00..10e6629d 100644 --- a/models/silver/core/tests/traces/test_silver__traces_recent.yml +++ b/models/silver/core/tests/traces/test_silver__traces_recent.yml @@ -1,35 +1,52 @@ version: 2 models: - name: test_silver__traces_recent + description: "This is a view used to test the last three days of traces." tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - - BLOCK_NUMBER - - TX_POSITION - - TRACE_INDEX + - TRACES_ID + columns: - name: BLOCK_NUMBER tests: - - not_null + - not_null + - name: TX_POSITION + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: TRACE_JSON + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP tests: - - not_null: - where: NOT IS_PENDING + - not_null - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + datepart: hour + interval: 2 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_LTZ - TIMESTAMP_NTZ - - name: TX_HASH + - name: MODIFIED_TIMESTAMP tests: - - not_null: - where: NOT IS_PENDING - - dbt_expectations.expect_column_values_to_match_regex: - regex: 0[xX][0-9a-fA-F]+ - \ No newline at end of file + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql index cb859be4..9e695af8 100644 --- a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__transactions') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days - ) + {{ ref("_block_lookback") }} + ) \ No newline at end of file diff --git a/models/silver/core/tests/transfers/test_silver__transfers_recent.sql b/models/silver/core/tests/transfers/test_silver__transfers_recent.sql index 2b847463..e384cdaa 100644 --- a/models/silver/core/tests/transfers/test_silver__transfers_recent.sql +++ b/models/silver/core/tests/transfers/test_silver__transfers_recent.sql @@ -3,21 +3,14 @@ tags = ['recent_test'] ) }} -WITH last_3_days AS ( - - SELECT - block_number - FROM - {{ ref("_block_lookback") }} -) SELECT * FROM {{ ref('silver__transfers') }} WHERE - block_number >= ( + block_number > ( SELECT block_number FROM - last_3_days + {{ ref("_block_lookback") }} ) diff --git a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql index b8e2f308..ec82fab2 100644 --- a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql +++ b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql @@ -1,12 +1,11 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = ["block_number", "tx_position", "trace_address"], - tags = ['traces_fix'] +{# {{ config ( +materialized = "incremental", +incremental_strategy = 'delete+insert', +unique_key = ["block_number", "tx_position", "trace_address"], +tags = ['traces_fix'] ) }} {% set batch_query %} - SELECT MAX(next_batch_id) AS next_batch_id FROM @@ -161,4 +160,4 @@ aggregated_errors AS ( prod_trace_succeeded FROM batch - CROSS JOIN final_errors + CROSS JOIN final_errors #} diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql index e3a532bc..6db3a7cf 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql @@ -13,10 +13,19 @@ WITH base_contracts AS ( block_timestamp, from_address, to_address AS contract_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE from_address = '0xaf5191b0de278c7286d6c7cc6ab6bb8a73ba2cd6' --Factory AND TYPE ILIKE 'create%' diff --git a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql index 1c92e256..7df4387a 100644 --- a/models/silver/defi/dex/curve/silver_dex__curve_pools.sql +++ b/models/silver/defi/dex/curve/silver_dex__curve_pools.sql @@ -14,15 +14,24 @@ WITH contract_deployments AS ( block_timestamp, from_address AS deployer_address, to_address AS contract_address, - _call_id, - _inserted_timestamp, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp, ROW_NUMBER() over ( ORDER BY contract_address ) AS row_num FROM {{ ref( - 'silver__traces' + 'core__fact_traces' ) }} WHERE -- curve contract deployers diff --git a/models/silver/defi/lending/moonwell/silver__moonwell_asset_details.sql b/models/silver/defi/lending/moonwell/silver__moonwell_asset_details.sql index 15d31123..c9322d8a 100644 --- a/models/silver/defi/lending/moonwell/silver__moonwell_asset_details.sql +++ b/models/silver/defi/lending/moonwell/silver__moonwell_asset_details.sql @@ -60,7 +60,7 @@ traces_pull AS ( ELSE NULL END AS asset_identifier FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} t WHERE tx_hash IN ( diff --git a/models/silver/defi/lending/morpho/silver__morpho_borrows.sql b/models/silver/defi/lending/morpho/silver__morpho_borrows.sql index 5bb04da9..9ec2c257 100644 --- a/models/silver/defi/lending/morpho/silver__morpho_borrows.sql +++ b/models/silver/defi/lending/morpho/silver__morpho_borrows.sql @@ -40,10 +40,19 @@ WITH traces AS ( ) AS shares, CONCAT('0x', SUBSTR(segmented_input [7] :: STRING, 25)) AS on_behalf_address, CONCAT('0x', SUBSTR(segmented_input [8] :: STRING, 25)) AS receiver_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} t WHERE to_address = '0xbbbbbbbbbb9cc5e90e3b3af64bdaf62c37eeffcb' --Morpho Blue @@ -66,12 +75,12 @@ tx_join AS ( tx.block_number, tx.tx_hash, tx.block_timestamp, - tx.from_address as origin_from_address, - tx.to_address as origin_to_address, + tx.from_address AS origin_from_address, + tx.to_address AS origin_to_address, tx.origin_function_signature, t.from_address, - t.to_address as contract_address, - tx.from_address as borrower_address, + t.to_address AS contract_address, + tx.from_address AS borrower_address, t.loan_token, t.collateral_token, t.amount, @@ -96,16 +105,19 @@ SELECT t.contract_address, loan_token AS market, amount AS amount_unadj, - amount / pow(10, C.token_decimals) AS amount, + amount / pow( + 10, + C.token_decimals + ) AS amount, C.token_symbol, C.token_decimals, borrower_address, - t.contract_address as lending_pool_contract, + t.contract_address AS lending_pool_contract, 'Morpho Blue' AS platform, 'base' AS blockchain, - _call_id as _id, + _call_id AS _id, t._inserted_timestamp FROM - tx_join t + tx_join t LEFT JOIN {{ ref('silver__contracts') }} C - ON c.contract_address = t.loan_token + ON C.contract_address = t.loan_token diff --git a/models/silver/defi/lending/morpho/silver__morpho_deposits.sql b/models/silver/defi/lending/morpho/silver__morpho_deposits.sql index 8f3e8212..332f6eb6 100644 --- a/models/silver/defi/lending/morpho/silver__morpho_deposits.sql +++ b/models/silver/defi/lending/morpho/silver__morpho_deposits.sql @@ -40,10 +40,19 @@ WITH traces AS ( ) AS shares, CONCAT('0x', SUBSTR(segmented_input [7] :: STRING, 25)) AS on_behalf_address, CONCAT('0x', SUBSTR(segmented_input [8] :: STRING, 25)) AS receiver_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} t WHERE to_address = '0xbbbbbbbbbb9cc5e90e3b3af64bdaf62c37eeffcb' --Morpho Blue @@ -66,12 +75,12 @@ tx_join AS ( tx.block_number, tx.tx_hash, tx.block_timestamp, - tx.from_address as origin_from_address, - tx.to_address as origin_to_address, + tx.from_address AS origin_from_address, + tx.to_address AS origin_to_address, tx.origin_function_signature, t.from_address, - t.to_address as contract_address, - tx.from_address as depositor_address, + t.to_address AS contract_address, + tx.from_address AS depositor_address, t.loan_token, t.collateral_token, t.amount, @@ -96,16 +105,19 @@ SELECT t.contract_address, loan_token AS market, amount AS amount_unadj, - amount / pow(10, C.token_decimals) AS amount, + amount / pow( + 10, + C.token_decimals + ) AS amount, C.token_symbol, C.token_decimals, depositor_address, - t.contract_address as lending_pool_contract, + t.contract_address AS lending_pool_contract, 'Morpho Blue' AS platform, 'base' AS blockchain, - _call_id as _id, + _call_id AS _id, t._inserted_timestamp FROM - tx_join t + tx_join t LEFT JOIN {{ ref('silver__contracts') }} C - ON c.contract_address = t.loan_token + ON C.contract_address = t.loan_token diff --git a/models/silver/defi/lending/morpho/silver__morpho_liquidations.sql b/models/silver/defi/lending/morpho/silver__morpho_liquidations.sql index 28d6b3a6..dd654e85 100644 --- a/models/silver/defi/lending/morpho/silver__morpho_liquidations.sql +++ b/models/silver/defi/lending/morpho/silver__morpho_liquidations.sql @@ -25,11 +25,24 @@ WITH traces AS ( CONCAT('0x', SUBSTR(segmented_input [2] :: STRING, 25)) AS oracle_address, CONCAT('0x', SUBSTR(segmented_input [3] :: STRING, 25)) AS irm_address, CONCAT('0x', SUBSTR(segmented_input [5] :: STRING, 25)) AS borrower, - ROW_NUMBER() OVER (PARTITION BY tx_hash ORDER BY trace_index) AS trace_index_order, - _call_id, - _inserted_timestamp + ROW_NUMBER() over ( + PARTITION BY tx_hash + ORDER BY + trace_index + ) AS trace_index_order, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE to_address = '0xbbbbbbbbbb9cc5e90e3b3af64bdaf62c37eeffcb' --Morpho Blue AND function_sig = '0xd8eabcb8' @@ -52,7 +65,11 @@ logs AS( l.block_number, l.block_timestamp, l.event_index, - ROW_NUMBER() OVER (PARTITION BY l.tx_hash ORDER BY l.event_index) AS event_index_order, + ROW_NUMBER() over ( + PARTITION BY l.tx_hash + ORDER BY + l.event_index + ) AS event_index_order, l.origin_from_address, l.origin_to_address, l.origin_function_signature, diff --git a/models/silver/defi/lending/morpho/silver__morpho_repayments.sql b/models/silver/defi/lending/morpho/silver__morpho_repayments.sql index 76a521ba..70f7eb5a 100644 --- a/models/silver/defi/lending/morpho/silver__morpho_repayments.sql +++ b/models/silver/defi/lending/morpho/silver__morpho_repayments.sql @@ -40,15 +40,24 @@ WITH traces AS ( ) AS shares, CONCAT('0x', SUBSTR(segmented_input [7] :: STRING, 25)) AS on_behalf_address, CONCAT('0x', SUBSTR(segmented_input [8] :: STRING, 25)) AS receiver_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE to_address = '0xbbbbbbbbbb9cc5e90e3b3af64bdaf62c37eeffcb' --Morpho Blue AND function_sig = '0x20b76e81' AND trace_status = 'SUCCESS' - AND tx_status = 'SUCCESS' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -65,12 +74,12 @@ tx_join AS ( tx.block_number, tx.tx_hash, tx.block_timestamp, - tx.from_address as origin_from_address, - tx.to_address as origin_to_address, + tx.from_address AS origin_from_address, + tx.to_address AS origin_to_address, tx.origin_function_signature, t.from_address, - t.to_address as contract_address, - tx.from_address as borrower_address, + t.to_address AS contract_address, + tx.from_address AS borrower_address, t.loan_token, t.collateral_token, t.amount, @@ -99,16 +108,16 @@ SELECT 10, C.token_decimals ) AS amount, - on_behalf_address as payer, + on_behalf_address AS payer, borrower_address, - t.contract_address as lending_pool_contract, + t.contract_address AS lending_pool_contract, C.token_symbol, C.token_decimals, - 'Morpho Blue' as platform, + 'Morpho Blue' AS platform, 'base' AS blockchain, - t._call_id as _id, + t._call_id AS _id, t._inserted_timestamp FROM - tx_join t + tx_join t LEFT JOIN {{ ref('silver__contracts') }} C - ON c.contract_address = t.loan_token \ No newline at end of file + ON C.contract_address = t.loan_token diff --git a/models/silver/defi/lending/morpho/silver__morpho_withdraws.sql b/models/silver/defi/lending/morpho/silver__morpho_withdraws.sql index 9797cf02..da1b5ea3 100644 --- a/models/silver/defi/lending/morpho/silver__morpho_withdraws.sql +++ b/models/silver/defi/lending/morpho/silver__morpho_withdraws.sql @@ -40,16 +40,25 @@ WITH traces AS ( ) AS shares, CONCAT('0x', SUBSTR(segmented_input [7] :: STRING, 25)) AS on_behalf_address, CONCAT('0x', SUBSTR(segmented_input [8] :: STRING, 25)) AS receiver_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} t WHERE to_address = '0xbbbbbbbbbb9cc5e90e3b3af64bdaf62c37eeffcb' --Morpho Blue AND function_sig = '0x5c2bea49' - AND trace_status = 'SUCCESS' - AND tx_status = 'SUCCESS' + AND trace_status = 'SUCCESS' + AND tx_status = 'SUCCESS' {% if is_incremental() %} AND _inserted_timestamp >= ( @@ -66,12 +75,12 @@ tx_join AS ( tx.block_number, tx.tx_hash, tx.block_timestamp, - tx.from_address as origin_from_address, - tx.to_address as origin_to_address, + tx.from_address AS origin_from_address, + tx.to_address AS origin_to_address, tx.origin_function_signature, t.from_address, - t.to_address as contract_address, - tx.from_address as depositor_address, + t.to_address AS contract_address, + tx.from_address AS depositor_address, t.loan_token, t.collateral_token, t.amount, @@ -101,15 +110,14 @@ SELECT C.token_decimals ) AS amount, depositor_address, - t.contract_address as lending_pool_contract, + t.contract_address AS lending_pool_contract, C.token_symbol, C.token_decimals, - 'Morpho Blue' as platform, + 'Morpho Blue' AS platform, 'base' AS blockchain, - t._call_id as _id, + t._call_id AS _id, t._inserted_timestamp FROM - tx_join t + tx_join t LEFT JOIN {{ ref('silver__contracts') }} C - ON c.contract_address = t.loan_token - + ON C.contract_address = t.loan_token diff --git a/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql b/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql index 302e1cda..bdbf94b4 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql @@ -56,7 +56,7 @@ traces_pull AS ( from_address AS token_address, to_address AS underlying_asset FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE tx_hash IN ( SELECT diff --git a/models/silver/nft/sales/silver__zeroex_sales.sql b/models/silver/nft/sales/silver__zeroex_sales.sql index 9526349f..f08a71d6 100644 --- a/models/silver/nft/sales/silver__zeroex_sales.sql +++ b/models/silver/nft/sales/silver__zeroex_sales.sql @@ -84,9 +84,8 @@ AND _inserted_timestamp >= ( FROM {{ this }} ) -and _inserted_timestamp >= dateadd('day', -14, current_date()) -{% endif %} -), +AND _inserted_timestamp >= DATEADD('day', -14, CURRENT_DATE()) +{% endif %}), token_transfers_raw AS ( SELECT tx_hash, @@ -125,9 +124,8 @@ AND _inserted_timestamp >= ( FROM {{ this }} ) -and _inserted_timestamp >= dateadd('day', -14, current_date()) -{% endif %} -), +AND _inserted_timestamp >= DATEADD('day', -14, CURRENT_DATE()) +{% endif %}), logs_token_raw AS ( SELECT *, @@ -155,7 +153,7 @@ eth_transfers_raw AS ( SELECT tx_hash, to_address AS seller_address, - eth_value, + VALUE AS eth_value, trace_index AS transfers_index, ROW_NUMBER() over ( PARTITION BY tx_hash, @@ -164,7 +162,7 @@ eth_transfers_raw AS ( trace_index ASC ) AS intra_tx_index FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_timestamp :: DATE >= '2023-08-01' AND tx_hash IN ( @@ -178,10 +176,10 @@ eth_transfers_raw AS ( AND identifier != 'CALL_ORIGIN' AND from_address = '0xdef1c0ded9bec7f1a1670819833240f027b25eff' AND trace_status = 'SUCCESS' - AND eth_value > 0 + AND VALUE > 0 {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX( _inserted_timestamp @@ -189,9 +187,8 @@ AND _inserted_timestamp >= ( FROM {{ this }} ) -and _inserted_timestamp >= dateadd('day', -14, current_date()) -{% endif %} -), +AND modified_timestamp >= DATEADD('day', -14, CURRENT_DATE()) +{% endif %}), logs_eth_raw AS ( SELECT *, @@ -333,9 +330,8 @@ AND _inserted_timestamp >= ( FROM {{ this }} ) -and _inserted_timestamp >= dateadd('day', -14, current_date()) -{% endif %} -) +AND _inserted_timestamp >= DATEADD('day', -14, CURRENT_DATE()) +{% endif %}) SELECT block_number, block_timestamp, diff --git a/models/silver/overflow/silver__overflowed_traces2.sql b/models/silver/overflow/silver__overflowed_traces.sql similarity index 95% rename from models/silver/overflow/silver__overflowed_traces2.sql rename to models/silver/overflow/silver__overflowed_traces.sql index 69f0f117..7c1dc58b 100644 --- a/models/silver/overflow/silver__overflowed_traces2.sql +++ b/models/silver/overflow/silver__overflowed_traces.sql @@ -1,11 +1,11 @@ --- depends_on: {{ ref('bronze__overflowed_traces2') }} +-- depends_on: {{ ref('bronze__overflowed_traces') }} {% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} {{ config ( materialized = "incremental", incremental_strategy = 'delete+insert', unique_key = ['block_number','tx_position'], cluster_by = ['modified_timestamp::DATE','partition_key'], - tags = ['overflowed_traces2'], + tags = ['overflowed_traces'], snowflake_warehouse = warehouse ) }} @@ -73,7 +73,7 @@ WITH bronze_overflowed_traces AS ( '_' ) AS trace_address_array FROM - {{ ref("bronze__overflowed_traces2") }} + {{ ref("bronze__overflowed_traces") }} GROUP BY block_number, tx_position, diff --git a/models/sources.yml b/models/sources.yml index d818bc1f..997e13c4 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -50,11 +50,6 @@ sources: schema: crosschain tables: - name: node_mapping - - name: base_silver - database: base - schema: silver - tables: - - name: verified_abis - name: base_bronze_api database: base schema: bronze_api @@ -74,4 +69,10 @@ sources: database: "{{ 'base' if target.database == 'BASE' else 'base_dev' }}" schema: silver tables: - - name: overflowed_traces2 \ No newline at end of file + - name: overflowed_traces + - name: verified_abis + - name: base_gold + database: "{{ 'base' if target.database == 'BASE' else 'base_dev' }}" + schema: core + tables: + - name: fact_traces diff --git a/models/streamline/silver/core/retry/_missing_traces.sql b/models/streamline/silver/core/retry/_missing_traces.sql index cb70a8f7..34f3aa25 100644 --- a/models/streamline/silver/core/retry/_missing_traces.sql +++ b/models/streamline/silver/core/retry/_missing_traces.sql @@ -7,7 +7,7 @@ SELECT FROM {{ ref("test_silver__transactions_recent") }} tx - LEFT JOIN {{ ref("test_silver__traces_recent") }} + LEFT JOIN {{ ref("test_gold__fact_traces_recent") }} tr USING ( block_number, tx_hash diff --git a/tests/base/test_silver__missing_traces.sql b/tests/base/test_gold__missing_traces.sql similarity index 52% rename from tests/base/test_silver__missing_traces.sql rename to tests/base/test_gold__missing_traces.sql index c96256f3..514d31cf 100644 --- a/tests/base/test_silver__missing_traces.sql +++ b/tests/base/test_gold__missing_traces.sql @@ -1,2 +1,2 @@ -- depends_on: {{ ref('test_silver__transactions_full') }} -{{ missing_txs(ref("test_silver__traces_full")) }} +{{ missing_txs(ref("test_gold__fact_traces_full")) }} diff --git a/tests/base/test_gold__recent_missing_traces.sql b/tests/base/test_gold__recent_missing_traces.sql new file mode 100644 index 00000000..9a571cae --- /dev/null +++ b/tests/base/test_gold__recent_missing_traces.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('test_silver__transactions_recent') }} +{{ recent_missing_txs(ref("test_gold__fact_traces_recent")) }} diff --git a/tests/base/test_silver__recent_missing_traces.sql b/tests/base/test_silver__recent_missing_traces.sql deleted file mode 100644 index 26a69d4b..00000000 --- a/tests/base/test_silver__recent_missing_traces.sql +++ /dev/null @@ -1,2 +0,0 @@ --- depends_on: {{ ref('test_silver__transactions_recent') }} -{{ recent_missing_txs(ref("test_silver__traces_recent")) }}