make macros balancer_compatible (duneanalytics#6098)

* make macros balancer_compatible * fix typo * test * again * again * again * oops * remove models from pr * update beethoven_x models * add new macros for beethoven x * fix schema * again * again * again * fix * again * revert beets liquidity * add protocol fees collector address for fantom * fix * fix upstream refs * retry * update bpt prices macro * filter to remove dupes --------- Co-authored-by: Huang Geyang <[email protected]> Co-authored-by: Alan Ghobadi <[email protected]>
viniabussafi · Jul 5, 2024 · 0ddb1e0 · 0ddb1e0
1 parent 8eb12f5
commit 0ddb1e0
Show file tree

Hide file tree

Showing 91 changed files with 676 additions and 888 deletions.
diff --git a/macros/models/_project/balancer/balancer_bpt_prices_macro.sql b/macros/models/_project/balancer/balancer_bpt_prices_macro.sql
@@ -1,6 +1,6 @@
 {% macro 
-    balancer_bpt_prices_macro(
-        blockchain, version
+    balancer_v2_compatible_bpt_prices_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
     ) 
 %}
 
@@ -9,7 +9,7 @@ WITH pool_labels AS (
             address AS pool_id,
             name AS pool_symbol,
             pool_type
-        FROM {{ source('labels', 'balancer_v2_pools') }}
+        FROM {{ pool_labels_spell }}
         WHERE blockchain = '{{blockchain}}'
     ),
 
@@ -48,7 +48,7 @@ WITH pool_labels AS (
                     poolId AS pool_id,
                     tokenIn AS token,
                     CAST(amountIn as int256) AS delta
-                FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_Swap') }}
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
 
                 UNION ALL
 
@@ -57,7 +57,7 @@ WITH pool_labels AS (
                     poolId AS pool_id,
                     tokenOut AS token,
                     -CAST(amountOut AS int256) AS delta
-                FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_Swap') }}
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
             ) swaps
         GROUP BY 1, 2, 3
     ),
@@ -69,7 +69,7 @@ WITH pool_labels AS (
             t.tokens,
             d.deltas,
             p.protocolFeeAmounts
-        FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_PoolBalanceChanged') }}
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceChanged') }}
         CROSS JOIN UNNEST (tokens) WITH ORDINALITY as t(tokens,i)
         CROSS JOIN UNNEST (deltas) WITH ORDINALITY as d(deltas,i)
         CROSS JOIN UNNEST (protocolFeeAmounts) WITH ORDINALITY as p(protocolFeeAmounts,i)
@@ -94,7 +94,7 @@ WITH pool_labels AS (
             poolId AS pool_id,
             token,
             cashDelta + managedDelta AS delta
-        FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_PoolBalanceManaged') }}
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceManaged') }}
     ),
 
     daily_delta_balance AS (
@@ -177,7 +177,7 @@ WITH pool_labels AS (
             ROW_NUMBER() OVER (partition by b.day, b.pool_id ORDER BY SUM(b.protocol_liquidity_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets
             SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity
         FROM cumulative_usd_balance b
-        LEFT JOIN {{ ref('balancer_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
+        LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
         AND b.token = w.token_address
         AND b.protocol_liquidity_usd > 0
         LEFT JOIN {{ ref('balancer_token_whitelist') }} q ON b.token = q.address 
@@ -208,7 +208,7 @@ WITH pool_labels AS (
     FROM cumulative_usd_balance c
     FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day
     AND c.pool_id = b.pool_id
-    LEFT JOIN {{ ref('balancer_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
+    LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
     AND w.blockchain = '{{blockchain}}'
     AND w.version = '{{version}}'    
     AND w.token_address = c.token
@@ -220,8 +220,8 @@ WITH pool_labels AS (
 
     bpt_trades AS (
         SELECT * 
-        FROM {{ source('balancer_v2_' ~ blockchain, 'Vault_evt_Swap') }} v
-        INNER JOIN pool_labels l ON bytearray_substring(v.poolId, 1, 20) = l.pool_id AND l.pool_type = 'linear'
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} v
+        LEFT JOIN pool_labels l ON bytearray_substring(v.poolId, 1, 20) = l.pool_id
         WHERE v.tokenIn = bytearray_substring(v.poolId, 1, 20) OR v.tokenOut = bytearray_substring(v.poolId, 1, 20)
     ), 
 
@@ -340,7 +340,7 @@ WITH pool_labels AS (
         ELSE l.liquidity / s.supply 
         END AS bpt_price
     FROM tvl l
-    LEFT JOIN {{ ref('balancer_bpt_supply') }} s ON l.pool_address = s.token_address
+    LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON l.pool_address = s.token_address
     AND l.blockchain = s.blockchain
     AND l.version = s.version  
     AND l.day = s.day

diff --git a/macros/models/_project/balancer/balancer_bpt_supply_macro.sql b/macros/models/_project/balancer/balancer_bpt_supply_macro.sql
@@ -1,6 +1,6 @@
 {% macro 
-    balancer_bpt_supply_macro(
-        blockchain, version
+    balancer_v2_compatible_bpt_supply_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
     ) 
 %}
 
@@ -11,7 +11,7 @@ WITH pool_labels AS (
                 name,
                 pool_type,
                 ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num
-            FROM {{ source('labels', 'balancer_v2_pools') }}
+            FROM {{ pool_labels_spell }}
             WHERE blockchain = '{{blockchain}}'
             GROUP BY 1, 2, 3) 
         WHERE num = 1
@@ -24,7 +24,7 @@ WITH pool_labels AS (
             contract_address AS token,
             COALESCE(SUM(CASE WHEN t."from" = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS mints,
             COALESCE(SUM(CASE WHEN t.to = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS burns
-        FROM  {{ ref('balancer_transfers_bpt') }} t
+        FROM  {{ ref(base_spells_namespace + '_transfers_bpt') }} t
         WHERE blockchain = '{{blockchain}}'   
         AND version = '{{version}}'
         GROUP BY 1, 2
@@ -47,7 +47,7 @@ WITH pool_labels AS (
             t.token,
             d.delta,
             ROW_NUMBER() OVER (PARTITION BY poolId ORDER BY evt_block_time ASC) AS rn
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolBalanceChanged') }} pb
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceChanged') }} pb
         CROSS JOIN UNNEST (pb.deltas) WITH ORDINALITY d(delta, i)
         CROSS JOIN UNNEST (pb.tokens) WITH ORDINALITY t(token, i)
         WHERE d.i = t.i 
@@ -85,7 +85,7 @@ WITH pool_labels AS (
             THEN 0
             ELSE SUM(amountOut / POWER(10, 18)) 
             END AS ajoins
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }} 
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} 
         LEFT JOIN pool_labels ON BYTEARRAY_SUBSTRING(poolId, 1, 20) = address
         WHERE tokenOut = BYTEARRAY_SUBSTRING(poolId, 1, 20)       
         GROUP BY 1, 2, 3
@@ -100,7 +100,7 @@ WITH pool_labels AS (
             THEN 0
             ELSE SUM(amountIn / POWER(10, 18)) 
             END AS aexits
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }} 
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} 
         LEFT JOIN pool_labels ON BYTEARRAY_SUBSTRING(poolId, 1, 20) = address
         WHERE tokenIn = BYTEARRAY_SUBSTRING(poolId, 1, 20)        
         GROUP BY 1, 2, 3

diff --git a/macros/models/_project/balancer/balancer_liquidity_macro.sql b/macros/models/_project/balancer/balancer_liquidity_macro.sql
@@ -1,6 +1,6 @@
 {% macro 
-    balancer_liquidity_macro(
-        blockchain, version
+    balancer_v2_compatible_liquidity_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
     ) 
 %}
 
@@ -9,7 +9,7 @@ WITH pool_labels AS (
             address AS pool_id,
             name AS pool_symbol,
             pool_type
-        FROM {{ source('labels', 'balancer_v2_pools') }}
+        FROM {{ pool_labels_spell }}
         WHERE blockchain = '{{blockchain}}'
     ),
 
@@ -60,7 +60,7 @@ WITH pool_labels AS (
             contract_address AS token,
             decimals,
             bpt_price
-        FROM {{ ref('balancer_bpt_prices') }}
+        FROM {{ ref(base_spells_namespace + '_bpt_prices') }}
         WHERE blockchain = '{{blockchain}}'
         AND version = '{{version}}'
     ),
@@ -96,7 +96,7 @@ WITH pool_labels AS (
                     poolId AS pool_id,
                     tokenIn AS token,
                     CAST(amountIn as int256) AS delta
-                FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }}
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
 
                 UNION ALL
 
@@ -105,7 +105,7 @@ WITH pool_labels AS (
                     poolId AS pool_id,
                     tokenOut AS token,
                     -CAST(amountOut AS int256) AS delta
-                FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }}
+                FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
             ) swaps
         GROUP BY 1, 2, 3
     ),
@@ -117,7 +117,7 @@ WITH pool_labels AS (
             t.tokens,
             d.deltas,
             p.protocolFeeAmounts
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolBalanceChanged') }}
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceChanged') }}
         CROSS JOIN UNNEST (tokens) WITH ORDINALITY as t(tokens,i)
         CROSS JOIN UNNEST (deltas) WITH ORDINALITY as d(deltas,i)
         CROSS JOIN UNNEST (protocolFeeAmounts) WITH ORDINALITY as p(protocolFeeAmounts,i)
@@ -142,7 +142,7 @@ WITH pool_labels AS (
             poolId AS pool_id,
             token,
             cashDelta + managedDelta AS delta
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolBalanceManaged') }}
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceManaged') }}
     ),
 
     daily_delta_balance AS (
@@ -235,7 +235,7 @@ WITH pool_labels AS (
             SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity,
             SUM(b.pool_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1)  AS pool_liquidity
         FROM cumulative_usd_balance b
-        LEFT JOIN {{ ref('balancer_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
+        LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
         AND b.token = w.token_address
         AND b.pool_liquidity_usd > 0
         LEFT JOIN {{ ref('balancer_token_whitelist') }} q ON b.token = q.address 
@@ -276,7 +276,7 @@ WITH pool_labels AS (
     FROM cumulative_usd_balance c
     FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day
     AND c.pool_id = b.pool_id
-    LEFT JOIN {{ ref('balancer_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
+    LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id 
     AND w.blockchain = '{{blockchain}}'
     AND w.version = '{{version}}'     
     AND w.token_address = c.token

diff --git a/macros/models/_project/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql b/macros/models/_project/balancer/balancer_pool_token_supply_changes_daily_agg_macro.sql
@@ -1,6 +1,6 @@
 {% macro 
-    bpt_supply_changes_daily_agg_macro(
-        blockchain, version
+    balancer_v2_compatible_bpt_supply_changes_daily_agg_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace
     ) 
 %}
 WITH
@@ -13,7 +13,7 @@ WITH
             token_address,
             LEAD(block_date, 1, NOW()) OVER (PARTITION BY token_address ORDER BY block_date) AS day_of_next_change,
             SUM(delta_amount) AS daily_amount
-        FROM {{ ref('balancer_bpt_supply_changes') }}
+        FROM {{ ref(base_spells_namespace + '_bpt_supply_changes') }}
         WHERE blockchain = '{{blockchain}}'
         GROUP BY 1, 2, 3, 4, 5
     ),
@@ -34,4 +34,5 @@ WITH
         FROM calendar c
         LEFT JOIN daily_balance b ON b.block_date = c.day
         WHERE b.token_address IS NOT NULL
+        AND b.pool_type IS NOT NULL
     {% endmacro %}
diff --git a/macros/models/_project/balancer/balancer_pool_token_supply_changes_macro.sql b/macros/models/_project/balancer/balancer_pool_token_supply_changes_macro.sql
@@ -1,18 +1,18 @@
 {% macro 
-    bpt_supply_changes_macro(
-        blockchain, version
+    balancer_v2_compatible_bpt_supply_changes_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
     ) 
 %}
 WITH pool_labels AS (
         SELECT
             address,
             name,
             pool_type
-        FROM {{ source('labels', 'balancer_v2_pools') }}
+        FROM {{ pool_labels_spell }}
         WHERE blockchain = '{{blockchain}}'
     ),
 
-      -- Extract mints and burns from transfers
+    -- Extract mints and burns from transfers
     transfers AS (
         SELECT
             t.evt_block_time,
@@ -35,7 +35,7 @@ WITH pool_labels AS (
                 THEN - value
                 ELSE 0
                 END AS amount
-        FROM {{ ref('balancer_transfers_bpt') }} t
+        FROM {{ ref(base_spells_namespace + '_transfers_bpt') }} t
         LEFT JOIN pool_labels l ON t.contract_address = l.address
         WHERE t.blockchain = '{{blockchain}}'
         AND t.version = '{{version}}'
@@ -59,7 +59,7 @@ WITH pool_labels AS (
             THEN 0
             ELSE s.amountOut 
             END AS amount
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }} s
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} s
         LEFT JOIN pool_labels l ON BYTEARRAY_SUBSTRING(s.poolId, 1, 20) = l.address
         WHERE tokenOut = BYTEARRAY_SUBSTRING(s.poolId, 1, 20)
         {% if is_incremental() %}
@@ -82,7 +82,7 @@ WITH pool_labels AS (
             THEN 0
             ELSE - s.amountIn
             END AS amount
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_Swap') }} s
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }} s
         LEFT JOIN pool_labels l ON BYTEARRAY_SUBSTRING(s.poolId, 1, 20) = l.address
         WHERE tokenIn = BYTEARRAY_SUBSTRING(s.poolId, 1, 20)
         {% if is_incremental() %}

diff --git a/macros/models/_project/balancer/balancer_protocol_fee_macro.sql b/macros/models/_project/balancer/balancer_protocol_fee_macro.sql
@@ -1,6 +1,6 @@
 {% macro 
-    balancer_protocol_fee_macro(
-        blockchain, version
+    balancer_v2_compatible_protocol_fee_macro(
+        blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
     ) 
 %}
 
@@ -11,7 +11,7 @@ WITH pool_labels AS (
                 name,
                 pool_type,
                 ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num
-            FROM {{ source('labels', 'balancer_v2_pools') }}
+            FROM {{ pool_labels_spell }}
             WHERE blockchain = '{{blockchain}}'
             GROUP BY 1, 2, 3) 
         WHERE num = 1
@@ -65,8 +65,8 @@ WITH pool_labels AS (
             s.token_address AS token,
             18 AS decimals,
             SUM(protocol_liquidity_usd / supply) AS price
-        FROM {{ ref('balancer_liquidity') }} l
-        LEFT JOIN {{ ref('balancer_bpt_supply') }} s ON s.token_address = l.pool_address 
+        FROM {{ ref(base_spells_namespace + '_liquidity') }} l
+        LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON s.token_address = l.pool_address 
         AND l.blockchain = s.blockchain AND s.day = l.day AND s.supply > 0
         WHERE l.blockchain = '{{blockchain}}'
         AND l.version = '{{version}}'
@@ -89,7 +89,7 @@ WITH pool_labels AS (
             poolId AS pool_id,
             token AS token_address,
             SUM(protocol_fees) AS protocol_fee_amount_raw
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolBalanceChanged') }} b
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolBalanceChanged') }} b
         CROSS JOIN unnest("protocolFeeAmounts", "tokens") AS t(protocol_fees, token)   
         GROUP BY 1, 2, 3 
 
@@ -100,11 +100,15 @@ WITH pool_labels AS (
             poolId AS pool_id,
             b.poolAddress AS token_address,
             sum(value) AS protocol_fee_amount_raw
-        FROM {{ source('balancer_v2_' + blockchain, 'Vault_evt_PoolRegistered') }} b
+        FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }} b
         INNER JOIN {{ source('erc20_' + blockchain, 'evt_transfer') }} t
             ON t.contract_address = b.poolAddress
             AND t."from" = 0x0000000000000000000000000000000000000000
-            AND t.to = 0xce88686553686DA562CE7Cea497CE749DA109f9F --ProtocolFeesCollector address, which is the same across all chains   
+            AND t."to" =
+                CASE
+                    WHEN '{{blockchain}}' = 'fantom' THEN 0xc6920d3a369e7c8bd1a22dbe385e11d1f7af948f
+                    ELSE 0xce88686553686DA562CE7Cea497CE749DA109f9F
+                    END --ProtocolFeesCollector address, which is the same across all chains except for fantom   
         GROUP BY 1, 2, 3
     ),