AN-5227/add-poylmarket-filled-orders (#395)

* filled orders silver

* commit stuff

* fix maker

* add timestamps and

* format

models/silver/protocols/polymarket/silver__polymarket_filled_orders.sql

@@ -0,0 +1,226 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = "block_number",
+    cluster_by = ['block_timestamp::DATE'],
+    tags = ['curated','reorg']
+) }}
+
+WITH polymarket_orders AS(
+
+    SELECT
+        tx_hash,
+        block_number,
+        block_timestamp,
+        event_index,
+        origin_from_address,
+        origin_to_address,
+        origin_function_signature,
+        contract_address,
+        regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data,
+        segmented_data [1] :: STRING AS order_hash,
+        LOWER(CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40))) AS maker,
+        LOWER(CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40))) AS taker,
+        utils.udf_hex_to_int(
+            segmented_data [1] :: STRING
+        ) AS maker_asset_id,
+        utils.udf_hex_to_int(
+            segmented_data [1] :: STRING
+        ) AS taker_asset_id,
+        utils.udf_hex_to_int(
+            segmented_data [2] :: STRING
+        ) AS maker_amount_filled,
+        utils.udf_hex_to_int(
+            segmented_data [3] :: STRING
+        ) AS taker_amount_filled,
+        _inserted_timestamp,
+        _log_id
+    FROM
+        {{ ref('silver__logs') }}
+    WHERE
+        topics [0] :: STRING = '0xd0a08e8c493f9c94f29311604c9de1b4e8c8d4c06bd0c789af57f2d65bfec0f6'
+    AND contract_address in (
+        LOWER('0x4bfb41d5b3570defd03c39a9a4d8de6bd8b8982e'),
+        LOWER('0xC5d563A36AE78145C45a50134d48A1215220f80a')
+    )
+    AND 
+        origin_function_signature IN ('0xe60f0c05', '0xd2539b37')
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        ) - INTERVAL '{{ var("LOOKBACK", "4 hours") }}'
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+polymarket_shape AS(
+    SELECT
+        tx_hash,
+        block_number,
+        block_timestamp,
+        event_index,
+        origin_from_address,
+        origin_to_address,
+        origin_function_signature,
+        contract_address,
+        order_hash,
+        maker,
+        taker,
+        COALESCE(NULLIF(maker_asset_id, '0'), taker_asset_id) AS asset_id, 
+        maker_asset_id,
+        taker_asset_id,
+        maker_amount_filled / pow(
+            10,
+            6
+        ) AS amount_usd,
+        taker_amount_filled / pow(
+            10,
+            6
+        ) AS shares,
+        maker_amount_filled/taker_amount_filled AS price_per_share,
+        _inserted_timestamp,
+        _log_id
+    FROM
+        polymarket_orders
+),
+yes_tokens AS(
+    SELECT
+        tx_hash,
+        block_number,
+        block_timestamp,
+        event_index,
+        'OrderFilled' AS event_name,
+        origin_from_address,
+        origin_to_address,
+        origin_function_signature,
+        contract_address,
+        order_hash,
+        maker,
+        taker,
+        condition_id,
+        question_id,
+        question,
+        market_slug,
+        end_date_iso,
+        token_1_outcome as outcome,
+        asset_id, 
+        maker_asset_id, 
+        taker_asset_id,
+        amount_usd,
+        shares,
+        price_per_share,
+        _inserted_timestamp,
+        _log_id
+    FROM
+        polymarket_shape p
+        INNER JOIN {{ source('external_polymarket','dim_markets') }} m
+        ON asset_id = token_1_token_id
+),
+no_tokens AS(
+
+    SELECT
+        block_number,
+        block_timestamp,
+        origin_from_address,
+        origin_to_address,
+        origin_function_signature,
+        tx_hash,
+        event_index,
+        'OrderFilled' AS event_name,
+        contract_address,
+        order_hash,
+        maker,
+        taker,
+        condition_id,
+        question,
+        market_slug,
+        end_date_iso,
+        token_2_outcome as outcome,
+        question_id,
+        asset_id, 
+        maker_asset_id, 
+        taker_asset_id,
+        amount_usd,
+        shares,
+        price_per_share,
+        _inserted_timestamp,
+        _log_id
+    FROM
+        polymarket_shape p
+        INNER JOIN {{ source('external_polymarket','dim_markets') }} m
+        ON asset_id = token_2_token_id
+)
+SELECT
+    block_number,
+    block_timestamp,
+    origin_from_address,
+    origin_to_address,
+    origin_function_signature,
+    tx_hash,
+    event_index,
+    event_name,
+    contract_address,
+    question,
+    market_slug,
+    end_date_iso,
+    outcome,
+    order_hash,
+    maker,
+    taker,
+    condition_id,
+    question_id,
+    asset_id, 
+    maker_asset_id, 
+    taker_asset_id,
+    amount_usd,
+    shares,
+    price_per_share,
+    _inserted_timestamp,
+    _log_id,
+    {{dbt_utils.generate_surrogate_key(
+        ['tx_hash','event_index']
+    )}} AS polymarket_filled_orders_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM    no_tokens
+UNION ALL
+SELECT
+    block_number,
+    block_timestamp,
+    origin_from_address,
+    origin_to_address,
+    origin_function_signature,
+    tx_hash,
+    event_index,
+    event_name,
+    contract_address,
+    question,
+    market_slug,
+    end_date_iso,
+    outcome,
+    order_hash,
+    maker,
+    taker,
+    condition_id,
+    question_id,
+    asset_id, 
+    maker_asset_id, 
+    taker_asset_id,
+    amount_usd,
+    shares,
+    price_per_share,
+    _inserted_timestamp,
+    _log_id,
+    {{ dbt_utils.generate_surrogate_key(
+        ['tx_hash','event_index']
+    ) }} AS polymarket_filled_orders_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    yes_tokens

models/silver/protocols/polymarket/silver__polymarket_filled_orders.yml

@@ -0,0 +1,70 @@
+version: 2
+models:
+  - name: silver__polymarket_filled_orders
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - _LOG_ID
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 3
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        tests:
+          - not_null
+      - name: ORIGIN_FROM_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: ORIGIN_TO_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TX_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: EVENT_INDEX
+        tests:
+          - not_null
+      - name: EVENT_NAME
+        tests:
+          - not_null
+      - name: CONTRACT_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: AMOUNT_USD
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL
+                - FLOAT
+                - NUMBER
+      - name: SHARES
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL
+                - FLOAT
+                - NUMBER
+      - name: PRICE_PER_SHARE
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - DECIMAL
+                - FLOAT
+                - NUMBER    

models/sources.yml

@@ -69,4 +69,9 @@ sources:
     database: external
     schema: defillama
     tables:
-      - name: dim_chains
+      - name: dim_chains
+  - name: external_polymarket
+    database: external
+    schema: polymarket
+    tables:
+      - name: dim_markets