Make block ranges consistent (take2) (#330)

This PR is based heavily on PR #329 but proposes a slightly different convention. Specifically, it assumes that we compute a `start` and `end` block for the accounting period, and both `start` and `end` block are included in the accounting, as it seems more natural to the author. E.g., this means that for consecutive weeks, we have week 1: [a,b] week 2: [b+1, c] week 3: [c+1, d] etc. All queries (except for the vouch registry one) have been updated in the form of forks. The plan would be to test this branch locally in the upcoming payouts of Jan 9, and compare results (or alternatively, use this branch in the dry-run script) Once we are convinced that all changes are correct, we would then need to update the original Dune queries so that we continue using the same query ids. --------- Co-authored-by: Felix Henneke <[email protected]>
cowprotocol · Jan 10, 2024 · 841ad72 · 841ad72
1 parent 7571656
commit 841ad72
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 28 deletions.
diff --git a/queries/dune_v2/period_block_interval.sql b/queries/dune_v2/period_block_interval.sql
@@ -1,6 +1,7 @@
--- V3: https://dune.com/queries/1541504
+-- https://github.com/cowprotocol/solver-rewards/pull/330
+-- V3: https://dune.com/queries/3333356
 select
     min("number") as start_block,
     max("number") as end_block
 from ethereum.blocks
-where date_trunc('minute', time) between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+where time >= cast('{{StartTime}}' as timestamp) and time < cast('{{EndTime}}' as timestamp)
diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql
@@ -1,7 +1,14 @@
--- https://github.com/cowprotocol/solver-rewards/pull/327
--- Query Here: https://dune.com/queries/3093726
+-- https://github.com/cowprotocol/solver-rewards/pull/330
+-- Query Here: https://dune.com/queries/3333368
 with
-batch_meta as (
+block_range as (
+    select
+        min("number") as start_block,
+        max("number") as end_block
+    from ethereum.blocks
+    where time >= cast('{{StartTime}}' as timestamp) and time < cast('{{EndTime}}' as timestamp)
+  )
+,batch_meta as (
     select b.block_time,
            b.block_number,
            b.tx_hash,
@@ -14,7 +21,7 @@ batch_meta as (
            num_trades,
            b.solver_address
     from cow_protocol_ethereum.batches b
-    where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+    where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range)
     and (b.solver_address = from_hex('{{SolverAddress}}') or '{{SolverAddress}}' = '0x')
     and (b.tx_hash = from_hex('{{TxHash}}') or '{{TxHash}}' = '0x')
 )
@@ -38,8 +45,8 @@ batch_meta as (
     left outer join cow_protocol_ethereum.order_rewards f
         on f.tx_hash = t.tx_hash
         and f.order_uid = t.order_uid
-    where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
-    and t.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+    where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range)
+    and t.block_number >= (select start_block from block_range) and t.block_number <= (select end_block from block_range)
     and (b.solver_address = from_hex('{{SolverAddress}}') or '{{SolverAddress}}' = '0x')
     and (t.tx_hash = from_hex('{{TxHash}}') or '{{TxHash}}' = '0x')
 )
@@ -89,7 +96,7 @@ batch_meta as (
                 and evt_tx_hash = b.tx_hash
              inner join batchwise_traders bt
                 on evt_tx_hash = bt.tx_hash
-    where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+    where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range)
       and 0x9008d19f58aabd9ed0d60971565aa8510560ab41 in (to, "from")
       and not contains(traders_in, "from")
       and not contains(traders_out, to)
@@ -225,12 +232,6 @@ incoming_and_outgoing as (
     SELECT from_hex(address_str) as address
     FROM ( VALUES {{TokenList}} ) as _ (address_str)
 )
-,block_range as (
-  select min(number) as start_block,
-       max(number) as end_block
-  from ethereum.blocks
-  where time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
-)
 ,internalized_imbalances as (
   select  b.block_time,
           b.tx_hash,
@@ -246,7 +247,7 @@ incoming_and_outgoing as (
     join tokens.erc20 t
         on contract_address = from_hex(token)
         and blockchain = 'ethereum'
-    where i.block_number between (select start_block from block_range) and (select end_block from block_range)
+    where i.block_number >= (select start_block from block_range) and i.block_number <= (select end_block from block_range)
     and ('{{SolverAddress}}' = '0x' or b.solver_address = from_hex('{{SolverAddress}}'))
     and ('{{TxHash}}' = '0x' or b.tx_hash = from_hex('{{TxHash}}'))
 )
@@ -316,7 +317,7 @@ incoming_and_outgoing as (
             date_trunc('hour', block_time) as hour,
             usd_value / units_bought as price
         FROM cow_protocol_ethereum.trades
-        WHERE block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+        WHERE block_number >= (select start_block from block_range) and block_number <= (select end_block from block_range)
         AND units_bought > 0
     UNION
         select
@@ -325,7 +326,7 @@ incoming_and_outgoing as (
             date_trunc('hour', block_time) as hour,
             usd_value / units_sold as price
         FROM cow_protocol_ethereum.trades
-        WHERE block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp)
+        WHERE block_number >= (select start_block from block_range) and block_number <= (select end_block from block_range)
         AND units_sold > 0
     ) as combined
     GROUP BY hour, contract_address, decimals
@@ -407,4 +408,4 @@ incoming_and_outgoing as (
         solver_address,
         concat(environment, '-', name)
 )
-select * from {{CTE_NAME}}
+select * from {{CTE_NAME}}
diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql
@@ -18,14 +18,14 @@ WITH observed_settlements AS (SELECT
                                             AND s.tx_nonce = at.tx_nonce
                                      JOIN settlement_scores ss
                                           ON at.auction_id = ss.auction_id
-                              WHERE ss.block_deadline > {{start_block}}
+                              WHERE ss.block_deadline >= {{start_block}}
                                 AND ss.block_deadline <= {{end_block}}),
 
      auction_participation as (SELECT ss.auction_id, array_agg(participant) as participating_solvers
                                FROM auction_participants
                                       JOIN settlement_scores ss
                                            ON auction_participants.auction_id = ss.auction_id
-                               WHERE block_deadline > {{start_block}}
+                               WHERE block_deadline >= {{start_block}}
                                  AND block_deadline <= {{end_block}}
                                GROUP BY ss.auction_id),
      reward_data AS (SELECT

diff --git a/queries/orderbook/quote_rewards.sql b/queries/orderbook/quote_rewards.sql
@@ -4,7 +4,7 @@ with winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as solver,
                                INNER JOIN orders o ON order_uid = uid
                                JOIN order_quotes oq ON t.order_uid = oq.order_uid
                         WHERE o.class = 'market'
-                          AND block_number BETWEEN {{start_block}} AND {{end_block}}
+                          AND block_number >= {{start_block}} AND block_number <= {{end_block}}
                           AND oq.solver != '\x0000000000000000000000000000000000000000')
 
 SELECT solver, count(*) AS num_quotes

diff --git a/src/queries.py b/src/queries.py
@@ -33,7 +33,7 @@ def with_params(self, params: list[QueryParameter]) -> Query:
     "PERIOD_BLOCK_INTERVAL": QueryData(
         name="Block Interval for Accounting Period",
         filepath="period_block_interval.sql",
-        q_id=1541504,
+        q_id=3333356,
     ),
     "VOUCH_REGISTRY": QueryData(
         name="Vouch Registry",
@@ -43,7 +43,7 @@ def with_params(self, params: list[QueryParameter]) -> Query:
     "PERIOD_SLIPPAGE": QueryData(
         name="Solver Slippage for Period",
         filepath="period_slippage.sql",
-        q_id=3093726,
+        q_id=3333368,
     ),
     "DASHBOARD_SLIPPAGE": QueryData(
         name="Period Solver Rewards",

diff --git a/src/scripts/gap_detector.py b/src/scripts/gap_detector.py
@@ -18,15 +18,15 @@
 
 DB_COUNT_QUERY = (
     "select count(*) as batches from settlements "
-    "where block_number between {{start}} and {{end}};"
+    "where block_number >= {{start}} and block_number <= {{end}};"
 )
 DB_HASH_QUERY = (
     "select concat('0x', encode(tx_hash, 'hex')) as tx_hash from settlements "
-    "where block_number between {{start}} and {{end}};"
+    "where block_number >= {{start}} and block_number <= {{end}};"
 )
 
-DUNE_COUNT_QUERY_ID = 2481826
-DUNE_HASH_QUERY_ID = 2532671
+DUNE_COUNT_QUERY_ID = 3333411
+DUNE_HASH_QUERY_ID = 3333413
 
 JANUARY_2023 = 16308190
 MAX_QUERYABLE_HASH_SET = 500