new columns (#105)

* new columns * remove internal columns from gold * amt * edits * SO * update pk * typo * tags * date * add event index and unique * tx fees * name * add contracts heal logic * add changes to gold * non null test * add non null test * add lookback * remove symbol * greatest date * coalesce * tx count * trade id * date --------- Co-authored-by: sam <[email protected]>
FlipsideCrypto · Nov 29, 2023 · 52b6b48 · 52b6b48
1 parent c06f91a
commit 52b6b48
Show file tree

Hide file tree

Showing 68 changed files with 1,065 additions and 258 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -50,7 +50,8 @@ query-comment:
 
 models:
   +copy_grants: true
-
+  +on_schema_change: "append_new_columns"
+
 # In this example config, we tell dbt to build all models in the example/ directory
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.

diff --git a/models/doc_descriptions/general/deprecation.md b/models/doc_descriptions/general/deprecation.md
@@ -0,0 +1,11 @@
+{% docs internal_column %}    
+
+Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024.
+
+{% enddocs %}
+
+{% docs amount_deprecation %}   
+
+This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`.
+
+{% enddocs %}
diff --git a/models/doc_descriptions/general/export_columns.md b/models/doc_descriptions/general/export_columns.md
@@ -0,0 +1,19 @@
+{% docs pk %}
+
+The unique identifier for each row in the table.
+
+{% enddocs %}
+
+{% docs inserted_timestamp %}
+
+The utc timestamp at which the row was inserted into the table.
+
+{% enddocs %}
+
+{% docs modified_timestamp %}
+
+The utc timestamp at which the row was last modified.
+
+{% enddocs %}
+
+
diff --git a/models/doc_descriptions/nft/nft_intra_event_index.md b/models/doc_descriptions/nft/nft_intra_event_index.md
@@ -0,0 +1,5 @@
+{% docs nft_intra_event_index %}
+
+The order of events within a single event index. This is primarily used for ERC1155 NFT batch transfer events. 
+
+{% enddocs %}
diff --git a/models/doc_descriptions/traces/base_traces_index.md b/models/doc_descriptions/traces/base_traces_index.md
@@ -0,0 +1,5 @@
+{% docs base_trace_index %}
+
+The index of the trace within the transaction.
+
+{% enddocs %}
diff --git a/models/gold/core/core__dim_contracts.sql b/models/gold/core/core__dim_contracts.sql
@@ -12,7 +12,15 @@ SELECT
     c0.block_number AS created_block_number,
     c0.block_timestamp AS created_block_timestamp,
     c0.tx_hash AS created_tx_hash,
-    c0.creator_address AS creator_address
+    c0.creator_address AS creator_address,
+    COALESCE (
+        c0.created_contracts_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['c0.created_contract_address']
+        ) }}
+    ) AS dim_contracts_id,
+    GREATEST(COALESCE(c0.inserted_timestamp, '2000-01-01'), COALESCE(c1.inserted_timestamp, '2000-01-01')) AS inserted_timestamp,
+    GREATEST(COALESCE(c0.modified_timestamp, '2000-01-01'), COALESCE(c1.modified_timestamp, '2000-01-01')) AS modified_timestamp
 FROM
     {{ ref('silver__created_contracts') }}
     c0

diff --git a/models/gold/core/core__dim_contracts.yml b/models/gold/core/core__dim_contracts.yml
@@ -19,4 +19,10 @@ models:
       - name: CREATED_TX_HASH
         description: 'The transaction hash when the contract was created'
       - name: CREATOR_ADDRESS
-        description: 'The address of the creator of the contract'
+        description: 'The address of the creator of the contract'
+      - name: DIM_CONTRACTS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'  
diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql
@@ -11,6 +11,20 @@ SELECT
     address_name,
     label_type,
     label_subtype,
-    project_name
+    project_name,
+    COALESCE (
+        labels_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['address']
+        ) }}
+    ) AS dim_labels_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__labels') }}
diff --git a/models/gold/core/core__dim_labels.yml b/models/gold/core/core__dim_labels.yml
@@ -50,4 +50,10 @@ models:
               column_type_list:
                 - STRING
                 - VARCHAR
+      - name: DIM_LABELS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'  
 
diff --git a/models/gold/core/core__ez_decoded_event_logs.sql b/models/gold/core/core__ez_decoded_event_logs.sql
@@ -20,7 +20,16 @@ SELECT
     topics,
     DATA,
     event_removed,
-    tx_status
+    tx_status,
+    COALESCE (
+        decoded_logs_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'event_index']
+        ) }}
+    ) AS ez_decoded_event_logs_id,
+    GREATEST(COALESCE(l.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp,
+    GREATEST(COALESCE(l.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp
 FROM
-    {{ ref('silver__decoded_logs') }} 
-    LEFT JOIN {{ ref('silver__contracts') }} using (contract_address)
+    {{ ref('silver__decoded_logs') }}
+    l
+    LEFT JOIN {{ ref('silver__contracts') }} C USING (contract_address)
diff --git a/models/gold/core/core__ez_decoded_event_logs.yml b/models/gold/core/core__ez_decoded_event_logs.yml
@@ -56,4 +56,10 @@ models:
       - name: EVENT_REMOVED
         description: '{{ doc("base_event_removed") }}' 
       - name: TX_STATUS
-        description: '{{ doc("base_tx_status") }}' 
+        description: '{{ doc("base_tx_status") }}' 
+      - name: EZ_DECODED_EVENT_LOGS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/core/core__ez_eth_transfers.sql b/models/gold/core/core__ez_eth_transfers.sql
@@ -1,104 +1,26 @@
 {{ config(
-    materialized = 'incremental',
-    incremental_strategy = 'delete+insert',
-    unique_key = 'block_number',
-    cluster_by = ['block_timestamp::DATE'],
-    tags = ['core','non_realtime','reorg'],
+    materialized = 'view',
     persist_docs ={ "relation": true,
     "columns": true }
 ) }}
 
-WITH eth_base AS (
-
-    SELECT
-        tx_hash,
-        block_number,
-        block_timestamp,
-        identifier,
-        from_address,
-        to_address,
-        eth_value,
-        _call_id,
-        _inserted_timestamp,
-        eth_value_precise_raw,
-        eth_value_precise,
-        tx_position,
-        trace_index
-    FROM
-        {{ ref('silver__traces') }}
-    WHERE
-        eth_value > 0
-        AND tx_status = 'SUCCESS'
-        AND trace_status = 'SUCCESS'
-        AND TYPE NOT IN (
-            'DELEGATECALL',
-            'STATICCALL'
-        )
-
-{% if is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp) - INTERVAL '72 hours'
-    FROM
-        {{ this }}
-)
-{% endif %}
-),
-tx_table AS (
-    SELECT
-        block_number,
-        tx_hash,
-        from_address AS origin_from_address,
-        to_address AS origin_to_address,
-        origin_function_signature
-    FROM
-        {{ ref('silver__transactions') }}
-    WHERE
-        tx_hash IN (
-            SELECT
-                DISTINCT tx_hash
-            FROM
-                eth_base
-        )
-
-{% if is_incremental() %}
-AND _inserted_timestamp >= (
-    SELECT
-        MAX(_inserted_timestamp) - INTERVAL '72 hours'
-    FROM
-        {{ this }}
-)
-{% endif %}
-)
 SELECT
-    tx_hash AS tx_hash,
-    block_number AS block_number,
-    block_timestamp AS block_timestamp,
-    identifier AS identifier,
+    tx_hash,
+    block_number,
+    block_timestamp,
+    identifier,
     origin_from_address,
     origin_to_address,
     origin_function_signature,
     from_address AS eth_from_address,
     to_address AS eth_to_address,
-    eth_value AS amount,
-    eth_value_precise_raw AS amount_precise_raw,
-    eth_value_precise AS amount_precise,
-    ROUND(
-        eth_value * price,
-        2
-    ) AS amount_usd,
+    amount,
+    amount_precise_raw,
+    amount_precise,
+    amount_usd,
     _call_id,
     _inserted_timestamp,
     tx_position,
     trace_index
 FROM
-    eth_base A
-    LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }}
-    ON DATE_TRUNC(
-        'hour',
-        A.block_timestamp
-    ) = HOUR
-    JOIN tx_table USING (
-        tx_hash,
-        block_number
-    )
+    {{ ref('silver__native_transfers') }}
diff --git a/models/gold/core/core__ez_eth_transfers.yml b/models/gold/core/core__ez_eth_transfers.yml
@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: core__ez_eth_transfers
-    description: '{{ doc("base_ez_eth_transfers_table_doc") }}'
+    description: 'Deprecating soon! Migrate to `core.ez_native_transfers` by Jan. 31 2024.'
 
     columns:
       - name: BLOCK_NUMBER

diff --git a/models/gold/core/core__ez_native_transfers.sql b/models/gold/core/core__ez_native_transfers.sql
@@ -0,0 +1,38 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    tx_hash,
+    block_number,
+    block_timestamp,
+    tx_position,
+    trace_index,
+    identifier,
+    origin_from_address,
+    origin_to_address,
+    origin_function_signature,
+    from_address,
+    to_address,
+    amount,
+    amount_precise_raw,
+    amount_precise,
+    amount_usd,
+    COALESCE (
+        native_transfers_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'trace_index']
+        ) }}
+    ) AS ez_native_transfers_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
+FROM
+    {{ ref('silver__native_transfers') }}
diff --git a/models/gold/core/core__ez_native_transfers.yml b/models/gold/core/core__ez_native_transfers.yml
@@ -0,0 +1,42 @@
+version: 2
+models:
+  - name: core__ez_native_transfers
+    description: '{{ doc("base_ez_eth_transfers_table_doc") }}'
+
+    columns:
+      - name: TX_HASH
+        description: '{{ doc("base_transfer_tx_hash") }}'
+      - name: BLOCK_NUMBER
+        description: '{{ doc("base_block_number") }}'
+      - name: BLOCK_TIMESTAMP
+        description: '{{ doc("base_block_timestamp") }}'
+      - name: TX_POSITION
+        description: '{{ doc("base_tx_position") }}'
+      - name: TRACE_INDEX
+        description: '{{ doc("base_trace_index") }}'
+      - name: IDENTIFIER
+        description: '{{ doc("base_traces_identifier") }}'
+      - name: ORIGIN_FROM_ADDRESS
+        description: '{{ doc("base_origin_from") }}'
+      - name: ORIGIN_TO_ADDRESS
+        description: '{{ doc("base_origin_to") }}'
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: '{{ doc("base_origin_sig") }}'
+      - name: FROM_ADDRESS
+        description: '{{ doc("base_transfer_from_address") }}'
+      - name: TO_ADDRESS
+        description: '{{ doc("base_transfer_to_address") }}'
+      - name: AMOUNT
+        description: '{{ doc("base_eth_amount") }}'
+      - name: AMOUNT_PRECISE_RAW
+        description: '{{ doc("precise_amount_unadjusted") }}'
+      - name: AMOUNT_PRECISE
+        description: '{{ doc("precise_amount_adjusted") }}'
+      - name: AMOUNT_USD
+        description: '{{ doc("base_eth_amount_usd") }}' 
+      - name: EZ_NATIVE_TRANSFERS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}' 
diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql
@@ -26,6 +26,20 @@ SELECT
     has_decimal,
     has_price,
     _log_id,
-    _inserted_timestamp
+    _inserted_timestamp,
+    COALESCE (
+        transfers_id,
+        {{ dbt_utils.generate_surrogate_key(
+            ['tx_hash', 'event_index']
+        ) }}
+    ) AS ez_token_transfers_id,
+    COALESCE(
+        inserted_timestamp,
+        '2000-01-01'
+    ) AS inserted_timestamp,
+    COALESCE(
+        modified_timestamp,
+        '2000-01-01'
+    ) AS modified_timestamp
 FROM
     {{ ref('silver__transfers') }}
diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml
@@ -45,4 +45,12 @@ models:
       - name: HAS_PRICE
         description: '{{ doc("base_transfer_has_price") }}'
       - name: _LOG_ID
-        description: '{{ doc("base_log_id_transfers") }}'
+        description: '{{ doc("internal_column") }}'
+      - name: _INSERTED_TIMESTAMP
+        description: '{{ doc("internal_column") }}'
+      - name: EZ_TOKEN_TRANSFERS_ID
+        description: '{{ doc("pk") }}'   
+      - name: INSERTED_TIMESTAMP
+        description: '{{ doc("inserted_timestamp") }}'   
+      - name: MODIFIED_TIMESTAMP
+        description: '{{ doc("modified_timestamp") }}'