From a35f5fb540ad3e23d3a26bedfb0774b3d32e4a1c Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:59:41 +0300 Subject: [PATCH] Simplify protocol fee calculations and remove consistency rewards (mirroring solver rewards changes) (#114) This PR mirrors the changes in the solver rewards repo from the following two PRs: - https://github.com/cowprotocol/solver-rewards/pull/401 - https://github.com/cowprotocol/solver-rewards/pull/404 Concretely, the calculation of protocol fees in the sql query is simplified by a lot, and consistency rewards are removed as they are no longer used. **Note**: some tests needed updating since the new way of fetching protocol fees is not available for the block range the test was set up for. The tests were updated accordingly, and one (bit strange) way to verify the numbers is to run the following dune query, as the data for that block range on Dune have been uploaded via the old (and NOT the proposed) and tested way. The tiny discrepancies are a result of rounding "errors", but they are very small. For the batch rewards test, this query should be used ``` select block_number, block_deadline, tx_hash, solver, data.execution_cost, data.surplus, data.protocol_fee, data.fee, data.uncapped_payment_eth, data.capped_payment, data.winning_score, data.reference_score from cowswap.raw_batch_rewards where block_deadline >= 20936269 and block_deadline <= 20936280 ``` while for the order rewards test, this query should be used ``` select block_number, order_uid, solver, data.quote_solver, tx_hash, data.surplus_fee, data.amount, data.protocol_fee, data.protocol_fee_token, data.protocol_fee_native_price, data.quote_sell_amount, data.quote_buy_amount, data.quote_gas_cost, data.quote_sell_token_price, data.partner_fee, data.partner_fee_recipient, data.protocol_fee_kind from cowswap.raw_order_rewards where block_number>= 20934809 and block_number <= 20934808 + 17 order by block_number ASC ``` As for the integrator fees test, this range on Dune should be user: `block_number>= 20934805 and block_number <= 20934809` --------- Co-authored-by: Felix Henneke --- src/sql/orderbook/barn_batch_rewards.sql | 348 ++++-------------- src/sql/orderbook/barn_order_rewards.sql | 381 ++++++-------------- src/sql/orderbook/prod_batch_rewards.sql | 348 ++++-------------- src/sql/orderbook/prod_order_rewards.sql | 379 ++++++-------------- tests/integration/test_fetch_orderbook.py | 417 +++++++++------------- 5 files changed, 530 insertions(+), 1343 deletions(-) diff --git a/src/sql/orderbook/barn_batch_rewards.sql b/src/sql/orderbook/barn_batch_rewards.sql index 2dcfa691..c75422ee 100644 --- a/src/sql/orderbook/barn_batch_rewards.sql +++ b/src/sql/orderbook/barn_batch_rewards.sql @@ -17,23 +17,6 @@ WITH observed_settlements AS ( ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), -auction_participation as ( - SELECT - ss.auction_id, - array_agg( - concat('0x', encode(participant, 'hex')) - ORDER BY - participant - ) as participating_solvers - FROM - auction_participants - JOIN settlement_scores ss ON auction_participants.auction_id = ss.auction_id - WHERE - block_deadline > {{start_block}} - AND block_deadline <= {{end_block}} - GROUP BY - ss.auction_id -), -- order data order_data AS ( SELECT @@ -56,10 +39,10 @@ order_data AS ( app_data FROM jit_orders ), --- additional trade data -order_surplus AS ( +-- unprocessed trade data +trade_data_unprocessed AS ( SELECT - ss.winner as solver, + ss.winner AS solver, s.auction_id, s.tx_hash, t.order_uid, @@ -67,21 +50,15 @@ order_surplus AS ( od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives - oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + oe.surplus_fee AS observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price od.kind, - CASE - WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * od.buy_amount / od.sell_amount - WHEN od.kind = 'buy' THEN t.buy_amount * od.sell_amount / od.buy_amount - t.sell_amount - END AS surplus, - CASE - WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN od.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount - END AS price_improvement, CASE WHEN od.kind = 'sell' THEN od.buy_token WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, - ad.full_app_data as app_data + convert_from(ad.full_app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' AS partner_fee_recipient, + COALESCE(oe.protocol_fee_amounts[1], 0) AS first_protocol_fee_amount, + COALESCE(oe.protocol_fee_amounts[2], 0) AS second_protocol_fee_amount FROM settlements s JOIN settlement_scores ss -- contains block_deadline @@ -93,282 +70,87 @@ order_surplus AS ( JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id - LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON od.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on od.app_data = ad.contract_app_data + ON od.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), --- protocol fees: -fee_policies_first_proxy as ( - select - auction_id, - order_uid, - max(application_order) as application_order, - count (*) as num_policies - from fee_policies - where auction_id in (select auction_id from order_surplus) - group by order_uid, auction_id -), -fee_policies_first as ( - select - fp.auction_id, - fp.order_uid, - fp.application_order, - fp.kind, - fp.surplus_factor, - fp.surplus_max_volume_factor, - fp.volume_factor, - fp.price_improvement_factor, - fp.price_improvement_max_volume_factor - from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order -), -fee_policies_temp as ( - select - * - from fee_policies - where auction_id in (select auction_id from order_surplus) - except (select * from fee_policies_first ) -), -fee_policies_second as ( - select - * - from fee_policies_temp - UNION - select - auction_id, - order_uid, - 0 as application_order, - 'volume' as kind, - null as surplus_factor, - null as surplus_max_volume_factor, - 0 as volume_factor, - null as price_improvement_factor, - null as price_improvement_max_volume_factor - from fee_policies_first_proxy where num_policies = 1 -), -order_protocol_fee_first AS ( +-- processed trade data: +trade_data_processed AS ( SELECT - os.auction_id, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.surplus, - os.price_improvement, - os.kind, - convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * os.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * os.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * os.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - fp.volume_factor / (1 - fp.volume_factor) * os.buy_amount - WHEN os.kind = 'buy' THEN - fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount - END - END AS protocol_fee_first, - os.surplus_token AS protocol_fee_token - FROM - order_surplus os - JOIN fee_policies_first fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid -), -order_surplus_intermediate as ( - select auction_id, + solver, + tx_hash, order_uid, + sell_amount, + buy_amount, + sell_token, + observed_fee, + surplus_token, + second_protocol_fee_amount, + first_protocol_fee_amount + second_protocol_fee_amount AS protocol_fee, + partner_fee_recipient, CASE - WHEN kind = 'sell' then sell_amount - ELSE sell_amount - protocol_fee_first - END as sell_amount, - CASE - WHEN kind = 'sell' then buy_amount + protocol_fee_first - ELSE buy_amount - END as buy_amount, - surplus + protocol_fee_first as surplus, - price_improvement + protocol_fee_first as price_improvement, - protocol_fee_kind_first, - protocol_fee_first, - partner_fee_recipient - from order_protocol_fee_first -), -order_protocol_fee as materialized ( - SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.sell_token, - os.observed_fee, - os.surplus, - os.surplus_token, - protocol_fee_kind_first, - fp.kind as protocol_fee_kind_second, - protocol_fee_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount - WHEN os.kind = 'buy' THEN - protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount - END - END AS protocol_fee, - osi.partner_fee_recipient, - CASE - WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + WHEN partner_fee_recipient IS NOT NULL THEN second_protocol_fee_amount ELSE 0 END AS partner_fee, - os.surplus_token AS protocol_fee_token + surplus_token AS protocol_fee_token FROM - order_surplus os - JOIN order_surplus_intermediate osi - ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id - JOIN fee_policies_second fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_unprocessed ), price_data AS ( SELECT - os.auction_id, - os.order_uid, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + tdp.auction_id, + tdp.order_uid, + ap_surplus.price / pow(10, 18) AS surplus_token_native_price, + ap_protocol.price / pow(10, 18) AS protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) AS network_fee_token_native_price FROM - order_surplus AS os + trade_data_processed AS tdp LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token - ON os.auction_id = ap_sell.auction_id - AND os.sell_token = ap_sell.token + ON tdp.auction_id = ap_sell.auction_id + AND tdp.sell_token = ap_sell.token LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token - ON os.auction_id = ap_surplus.auction_id - AND os.surplus_token = ap_surplus.token + ON tdp.auction_id = ap_surplus.auction_id + AND tdp.surplus_token = ap_surplus.token LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON os.auction_id = ap_protocol.auction_id - AND os.surplus_token = ap_protocol.token + ON tdp.auction_id = ap_protocol.auction_id + AND tdp.surplus_token = ap_protocol.token ), -combined_order_data AS ( +trade_data_processed_with_prices AS ( SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.surplus, - os.surplus_token, - opf.protocol_fee, - opf.protocol_fee_token, - CASE - WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second - ELSE opf.protocol_fee_kind_first - END AS protocol_fee_kind, - opf.partner_fee, - opf.partner_fee_recipient, + tdp.auction_id, + tdp.solver, + tdp.tx_hash, + tdp.order_uid, + tdp.surplus_token, + tdp.protocol_fee, + tdp.protocol_fee_token, + tdp.partner_fee, + tdp.partner_fee_recipient, CASE - WHEN os.sell_token != os.surplus_token THEN os.observed_fee - (os.sell_amount - os.observed_fee) / os.buy_amount * coalesce(opf.protocol_fee, 0) - ELSE os.observed_fee - coalesce(opf.protocol_fee, 0) + WHEN tdp.sell_token != tdp.surplus_token THEN tdp.observed_fee - (tdp.sell_amount - tdp.observed_fee) / tdp.buy_amount * COALESCE(tdp.protocol_fee, 0) + ELSE tdp.observed_fee - COALESCE(tdp.protocol_fee, 0) END AS network_fee, - os.sell_token as network_fee_token, + tdp.sell_token AS network_fee_token, surplus_token_native_price, protocol_fee_token_native_price, network_fee_token_native_price FROM - order_surplus AS os - LEFT OUTER JOIN order_protocol_fee as opf - ON os.auction_id = opf.auction_id - AND os.order_uid = opf.order_uid + trade_data_processed AS tdp JOIN price_data pd - ON os.auction_id = pd.auction_id - AND os.order_uid = pd.order_uid + ON tdp.auction_id = pd.auction_id + AND tdp.order_uid = pd.order_uid ), batch_protocol_fees AS ( SELECT solver, tx_hash, - sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee + sum(protocol_fee * protocol_fee_token_native_price) AS protocol_fee FROM - combined_order_data - group by + trade_data_processed_with_prices + GROUP BY solver, tx_hash ), @@ -376,10 +158,10 @@ batch_network_fees AS ( SELECT solver, tx_hash, - sum(network_fee * network_fee_token_native_price) as network_fee + sum(network_fee * network_fee_token_native_price) AS network_fee FROM - combined_order_data - group by + trade_data_processed_with_prices + GROUP BY solver, tx_hash ), @@ -390,7 +172,7 @@ reward_data AS ( ss.auction_id, -- TODO - Assuming that `solver == winner` when both not null -- We will need to monitor that `solver == winner`! - coalesce(os.solver, winner) as solver, + ss.winner as solver, block_number as settlement_block, block_deadline, coalesce(execution_cost, 0) as execution_cost, @@ -403,8 +185,6 @@ reward_data AS ( else 0 end as observed_score, reference_score, - -- auction_participation - participating_solvers, -- protocol_fees coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, coalesce( @@ -413,13 +193,13 @@ reward_data AS ( ) as network_fee FROM settlement_scores ss - -- If there are reported scores, - -- there will always be a record of auction participants - JOIN auction_participation ap ON ss.auction_id = ap.auction_id -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash LEFT OUTER JOIN batch_network_fees bnf ON bnf.tx_hash = os.tx_hash + WHERE + ss.block_deadline > {{start_block}} + AND ss.block_deadline <= {{end_block}} ), reward_per_auction as ( SELECT @@ -442,8 +222,7 @@ reward_per_auction as ( {{EPSILON_UPPER}} ) as capped_payment, winning_score, - reference_score, - participating_solvers as participating_solvers + reference_score FROM reward_data ) @@ -462,8 +241,7 @@ SELECT uncapped_payment :: text as uncapped_payment_eth, capped_payment :: text as capped_payment, winning_score :: text as winning_score, - reference_score :: text as reference_score, - participating_solvers + reference_score :: text as reference_score FROM reward_per_auction ORDER BY block_deadline diff --git a/src/sql/orderbook/barn_order_rewards.sql b/src/sql/orderbook/barn_order_rewards.sql index 5e12dbb8..96111de7 100644 --- a/src/sql/orderbook/barn_order_rewards.sql +++ b/src/sql/orderbook/barn_order_rewards.sql @@ -33,293 +33,145 @@ with trade_hashes as ( settlement.block_number > {{start_block}} and settlement.block_number <= {{end_block}} ), -order_surplus AS ( +-- order data +order_data AS ( SELECT - ss.winner as solver, + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM orders + UNION ALL + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM jit_orders +), +protocol_fee_kind AS ( + SELECT DISTINCT ON (fp.auction_id, fp.order_uid) + fp.auction_id, + fp.order_uid, + fp.kind + FROM fee_policies fp + JOIN trade_hashes th + ON fp.auction_id = th.auction_id AND fp.order_uid = th.order_uid + ORDER BY fp.auction_id, fp.order_uid, fp.application_order +), +-- unprocessed trade data +trade_data_unprocessed AS ( + SELECT + ss.winner AS solver, s.auction_id, s.tx_hash, t.order_uid, - o.sell_token, - o.buy_token, + od.sell_token, + od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives - oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price - o.kind, - CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) - WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount - END AS surplus, + oe.surplus_fee AS observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + od.kind, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN o.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount - END AS price_improvement, - CASE - WHEN o.kind = 'sell' THEN o.buy_token - WHEN o.kind = 'buy' THEN o.sell_token + WHEN od.kind = 'sell' THEN od.buy_token + WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, - ad.full_app_data as app_data + convert_from(ad.full_app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' AS partner_fee_recipient, + COALESCE(oe.protocol_fee_amounts[1], 0) AS first_protocol_fee_amount, + COALESCE(oe.protocol_fee_amounts[2], 0) AS second_protocol_fee_amount FROM settlements s JOIN settlement_scores ss -- contains block_deadline ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts - ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block - JOIN orders o -- contains tokens and limit amounts - ON t.order_uid = o.uid + ON s.block_number = t.block_number -- given the join that follows with the order execution table, this works even when multiple txs appear in the same block + JOIN order_data od -- contains tokens and limit amounts + ON t.order_uid = od.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id - LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON o.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on o.app_data = ad.contract_app_data + ON od.app_data = ad.contract_app_data WHERE - ss.block_deadline >= {{start_block}} - -- since this table filtered on block_deadline is joined with another table filtered on block_number - -- the bound for this table need to be a bit looser. - AND ss.block_deadline <= {{end_block}} + 100 -), -fee_policies_first_proxy as ( - select - auction_id, - order_uid, - max(application_order) as application_order, - count (*) as num_policies - from fee_policies - where auction_id in (select auction_id from order_surplus) - group by order_uid, auction_id -), -fee_policies_first as ( - select - fp.auction_id, - fp.order_uid, - fp.application_order, - fp.kind, - fp.surplus_factor, - fp.surplus_max_volume_factor, - fp.volume_factor, - fp.price_improvement_factor, - fp.price_improvement_max_volume_factor - from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order -), -fee_policies_temp as ( - select - * - from fee_policies - where auction_id in (select auction_id from order_surplus) - except (select * from fee_policies_first ) + s.block_number > {{start_block}} + AND s.block_number <= {{end_block}} ), -fee_policies_second as ( - select - * - from fee_policies_temp - UNION - select - auction_id, - order_uid, - 0 as application_order, - 'volume' as kind, - null as surplus_factor, - null as surplus_max_volume_factor, - 0 as volume_factor, - null as price_improvement_factor, - null as price_improvement_max_volume_factor - from fee_policies_first_proxy where num_policies = 1 -), -order_protocol_fee_first AS ( +-- processed trade data: +trade_data_processed AS ( SELECT - os.auction_id, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.surplus, - os.price_improvement, - os.kind, - convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind_first, + tdu.auction_id, + tdu.solver, + tdu.tx_hash, + tdu.order_uid, + tdu.sell_amount, + tdu.buy_amount, + tdu.sell_token, + tdu.observed_fee, + tdu.surplus_token, + tdu.second_protocol_fee_amount, + tdu.first_protocol_fee_amount + tdu.second_protocol_fee_amount AS protocol_fee, + tdu.partner_fee_recipient, CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * os.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * os.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * os.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - fp.volume_factor / (1 - fp.volume_factor) * os.buy_amount - WHEN os.kind = 'buy' THEN - fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount - END - END AS protocol_fee_first, - os.surplus_token AS protocol_fee_token + WHEN tdu.partner_fee_recipient IS NOT NULL THEN tdu.second_protocol_fee_amount + ELSE 0 + END AS partner_fee, + tdu.surplus_token AS protocol_fee_token, + pfk.kind as protocol_fee_kind FROM - order_surplus os - JOIN fee_policies_first fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_unprocessed tdu + LEFT OUTER JOIN protocol_fee_kind pfk + ON tdu.order_uid = pfk.order_uid AND tdu.auction_id = pfk.auction_id ), -order_surplus_intermediate as ( - select - auction_id, - order_uid, - CASE - WHEN kind = 'sell' then sell_amount - ELSE sell_amount - protocol_fee_first - END as sell_amount, - CASE - WHEN kind = 'sell' then buy_amount + protocol_fee_first - ELSE buy_amount - END as buy_amount, - surplus + protocol_fee_first as surplus, - price_improvement + protocol_fee_first as price_improvement, - protocol_fee_kind_first, - protocol_fee_first, - partner_fee_recipient - from order_protocol_fee_first -), -order_protocol_fee as ( +price_data AS ( SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.sell_token, - os.observed_fee, - os.surplus, - os.surplus_token, - protocol_fee_kind_first, - fp.kind as protocol_fee_kind_second, - protocol_fee_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount - WHEN os.kind = 'buy' THEN - protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount - END - END AS protocol_fee, - osi.partner_fee_recipient, - CASE - WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first - ELSE 0 - END AS partner_fee, - os.surplus_token AS protocol_fee_token + tdp.auction_id, + tdp.order_uid, + ap_surplus.price / pow(10, 18) AS surplus_token_native_price, + ap_protocol.price / pow(10, 18) AS protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) AS network_fee_token_native_price FROM - order_surplus os - JOIN order_surplus_intermediate osi - ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id - JOIN fee_policies_second fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_processed AS tdp + LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token + ON tdp.auction_id = ap_sell.auction_id + AND tdp.sell_token = ap_sell.token + LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token + ON tdp.auction_id = ap_surplus.auction_id + AND tdp.surplus_token = ap_surplus.token + LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON tdp.auction_id = ap_protocol.auction_id + AND tdp.surplus_token = ap_protocol.token ), -order_protocol_fee_prices AS ( +trade_data_processed_with_prices AS ( SELECT - opf.auction_id, - opf.solver, - opf.tx_hash, - opf.order_uid, - opf.surplus, - opf.protocol_fee, - opf.protocol_fee_token, - CASE - WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second - ELSE opf.protocol_fee_kind_first - END AS protocol_fee_kind, - opf.partner_fee, - opf.partner_fee_recipient, + tdp.auction_id, + tdp.solver, + tdp.tx_hash, + tdp.order_uid, + tdp.surplus_token, + tdp.protocol_fee, + tdp.protocol_fee_token, + tdp.partner_fee, + tdp.partner_fee_recipient, CASE - WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee - ELSE opf.protocol_fee - END AS network_fee_correction, - opf.sell_token as network_fee_token, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + WHEN tdp.sell_token != tdp.surplus_token THEN tdp.observed_fee - (tdp.sell_amount - tdp.observed_fee) / tdp.buy_amount * COALESCE(tdp.protocol_fee, 0) + ELSE tdp.observed_fee - COALESCE(tdp.protocol_fee, 0) + END AS network_fee, + tdp.sell_token AS network_fee_token, + surplus_token_native_price, + protocol_fee_token_native_price, + network_fee_token_native_price, + protocol_fee_kind FROM - order_protocol_fee as opf - JOIN auction_prices ap_sell -- contains price: sell token - ON opf.auction_id = ap_sell.auction_id - AND opf.sell_token = ap_sell.token - JOIN auction_prices ap_surplus -- contains price: surplus token - ON opf.auction_id = ap_surplus.auction_id - AND opf.surplus_token = ap_surplus.token - JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON opf.auction_id = ap_protocol.auction_id - AND opf.protocol_fee_token = ap_protocol.token + trade_data_processed AS tdp + JOIN price_data pd + ON tdp.auction_id = pd.auction_id + AND tdp.order_uid = pd.order_uid ), winning_quotes as ( SELECT @@ -351,7 +203,7 @@ winning_quotes as ( select trade_hashes.block_number as block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, - concat('0x', encode(oq.solver, 'hex')) as solver, + concat('0x', encode(trade_hashes.solver, 'hex')) as solver, quote_solver, concat('0x', encode(trade_hashes.tx_hash, 'hex')) as tx_hash, coalesce(surplus_fee, 0) :: text as surplus_fee, @@ -365,14 +217,15 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - cast(coalesce(opfp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, - opfp.partner_fee_recipient, - opfp.protocol_fee_kind + cast(coalesce(tdpwp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, + tdpwp.partner_fee_recipient, + tdpwp.protocol_fee_kind from trade_hashes left outer join order_execution o on trade_hashes.order_uid = o.order_uid and trade_hashes.auction_id = o.auction_id left outer join winning_quotes wq on trade_hashes.order_uid = wq.order_uid - left outer join order_protocol_fee_prices opfp on trade_hashes.order_uid = opfp.order_uid - and trade_hashes.auction_id = opfp.auction_id + left outer join trade_data_processed_with_prices tdpwp on trade_hashes.order_uid = tdpwp.order_uid + and trade_hashes.auction_id = tdpwp.auction_id left outer join order_quotes oq on trade_hashes.order_uid = oq.order_uid + order by trade_hashes.block_number asc diff --git a/src/sql/orderbook/prod_batch_rewards.sql b/src/sql/orderbook/prod_batch_rewards.sql index 2dcfa691..c75422ee 100644 --- a/src/sql/orderbook/prod_batch_rewards.sql +++ b/src/sql/orderbook/prod_batch_rewards.sql @@ -17,23 +17,6 @@ WITH observed_settlements AS ( ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), -auction_participation as ( - SELECT - ss.auction_id, - array_agg( - concat('0x', encode(participant, 'hex')) - ORDER BY - participant - ) as participating_solvers - FROM - auction_participants - JOIN settlement_scores ss ON auction_participants.auction_id = ss.auction_id - WHERE - block_deadline > {{start_block}} - AND block_deadline <= {{end_block}} - GROUP BY - ss.auction_id -), -- order data order_data AS ( SELECT @@ -56,10 +39,10 @@ order_data AS ( app_data FROM jit_orders ), --- additional trade data -order_surplus AS ( +-- unprocessed trade data +trade_data_unprocessed AS ( SELECT - ss.winner as solver, + ss.winner AS solver, s.auction_id, s.tx_hash, t.order_uid, @@ -67,21 +50,15 @@ order_surplus AS ( od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives - oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + oe.surplus_fee AS observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price od.kind, - CASE - WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * od.buy_amount / od.sell_amount - WHEN od.kind = 'buy' THEN t.buy_amount * od.sell_amount / od.buy_amount - t.sell_amount - END AS surplus, - CASE - WHEN od.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN od.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount - END AS price_improvement, CASE WHEN od.kind = 'sell' THEN od.buy_token WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, - ad.full_app_data as app_data + convert_from(ad.full_app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' AS partner_fee_recipient, + COALESCE(oe.protocol_fee_amounts[1], 0) AS first_protocol_fee_amount, + COALESCE(oe.protocol_fee_amounts[2], 0) AS second_protocol_fee_amount FROM settlements s JOIN settlement_scores ss -- contains block_deadline @@ -93,282 +70,87 @@ order_surplus AS ( JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id - LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON od.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on od.app_data = ad.contract_app_data + ON od.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), --- protocol fees: -fee_policies_first_proxy as ( - select - auction_id, - order_uid, - max(application_order) as application_order, - count (*) as num_policies - from fee_policies - where auction_id in (select auction_id from order_surplus) - group by order_uid, auction_id -), -fee_policies_first as ( - select - fp.auction_id, - fp.order_uid, - fp.application_order, - fp.kind, - fp.surplus_factor, - fp.surplus_max_volume_factor, - fp.volume_factor, - fp.price_improvement_factor, - fp.price_improvement_max_volume_factor - from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order -), -fee_policies_temp as ( - select - * - from fee_policies - where auction_id in (select auction_id from order_surplus) - except (select * from fee_policies_first ) -), -fee_policies_second as ( - select - * - from fee_policies_temp - UNION - select - auction_id, - order_uid, - 0 as application_order, - 'volume' as kind, - null as surplus_factor, - null as surplus_max_volume_factor, - 0 as volume_factor, - null as price_improvement_factor, - null as price_improvement_max_volume_factor - from fee_policies_first_proxy where num_policies = 1 -), -order_protocol_fee_first AS ( +-- processed trade data: +trade_data_processed AS ( SELECT - os.auction_id, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.surplus, - os.price_improvement, - os.kind, - convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * os.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * os.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * os.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - fp.volume_factor / (1 - fp.volume_factor) * os.buy_amount - WHEN os.kind = 'buy' THEN - fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount - END - END AS protocol_fee_first, - os.surplus_token AS protocol_fee_token - FROM - order_surplus os - JOIN fee_policies_first fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid -), -order_surplus_intermediate as ( - select auction_id, + solver, + tx_hash, order_uid, + sell_amount, + buy_amount, + sell_token, + observed_fee, + surplus_token, + second_protocol_fee_amount, + first_protocol_fee_amount + second_protocol_fee_amount AS protocol_fee, + partner_fee_recipient, CASE - WHEN kind = 'sell' then sell_amount - ELSE sell_amount - protocol_fee_first - END as sell_amount, - CASE - WHEN kind = 'sell' then buy_amount + protocol_fee_first - ELSE buy_amount - END as buy_amount, - surplus + protocol_fee_first as surplus, - price_improvement + protocol_fee_first as price_improvement, - protocol_fee_kind_first, - protocol_fee_first, - partner_fee_recipient - from order_protocol_fee_first -), -order_protocol_fee as materialized ( - SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.sell_token, - os.observed_fee, - os.surplus, - os.surplus_token, - protocol_fee_kind_first, - fp.kind as protocol_fee_kind_second, - protocol_fee_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount - WHEN os.kind = 'buy' THEN - protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount - END - END AS protocol_fee, - osi.partner_fee_recipient, - CASE - WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + WHEN partner_fee_recipient IS NOT NULL THEN second_protocol_fee_amount ELSE 0 END AS partner_fee, - os.surplus_token AS protocol_fee_token + surplus_token AS protocol_fee_token FROM - order_surplus os - JOIN order_surplus_intermediate osi - ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id - JOIN fee_policies_second fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_unprocessed ), price_data AS ( SELECT - os.auction_id, - os.order_uid, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + tdp.auction_id, + tdp.order_uid, + ap_surplus.price / pow(10, 18) AS surplus_token_native_price, + ap_protocol.price / pow(10, 18) AS protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) AS network_fee_token_native_price FROM - order_surplus AS os + trade_data_processed AS tdp LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token - ON os.auction_id = ap_sell.auction_id - AND os.sell_token = ap_sell.token + ON tdp.auction_id = ap_sell.auction_id + AND tdp.sell_token = ap_sell.token LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token - ON os.auction_id = ap_surplus.auction_id - AND os.surplus_token = ap_surplus.token + ON tdp.auction_id = ap_surplus.auction_id + AND tdp.surplus_token = ap_surplus.token LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON os.auction_id = ap_protocol.auction_id - AND os.surplus_token = ap_protocol.token + ON tdp.auction_id = ap_protocol.auction_id + AND tdp.surplus_token = ap_protocol.token ), -combined_order_data AS ( +trade_data_processed_with_prices AS ( SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.surplus, - os.surplus_token, - opf.protocol_fee, - opf.protocol_fee_token, - CASE - WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second - ELSE opf.protocol_fee_kind_first - END AS protocol_fee_kind, - opf.partner_fee, - opf.partner_fee_recipient, + tdp.auction_id, + tdp.solver, + tdp.tx_hash, + tdp.order_uid, + tdp.surplus_token, + tdp.protocol_fee, + tdp.protocol_fee_token, + tdp.partner_fee, + tdp.partner_fee_recipient, CASE - WHEN os.sell_token != os.surplus_token THEN os.observed_fee - (os.sell_amount - os.observed_fee) / os.buy_amount * coalesce(opf.protocol_fee, 0) - ELSE os.observed_fee - coalesce(opf.protocol_fee, 0) + WHEN tdp.sell_token != tdp.surplus_token THEN tdp.observed_fee - (tdp.sell_amount - tdp.observed_fee) / tdp.buy_amount * COALESCE(tdp.protocol_fee, 0) + ELSE tdp.observed_fee - COALESCE(tdp.protocol_fee, 0) END AS network_fee, - os.sell_token as network_fee_token, + tdp.sell_token AS network_fee_token, surplus_token_native_price, protocol_fee_token_native_price, network_fee_token_native_price FROM - order_surplus AS os - LEFT OUTER JOIN order_protocol_fee as opf - ON os.auction_id = opf.auction_id - AND os.order_uid = opf.order_uid + trade_data_processed AS tdp JOIN price_data pd - ON os.auction_id = pd.auction_id - AND os.order_uid = pd.order_uid + ON tdp.auction_id = pd.auction_id + AND tdp.order_uid = pd.order_uid ), batch_protocol_fees AS ( SELECT solver, tx_hash, - sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee + sum(protocol_fee * protocol_fee_token_native_price) AS protocol_fee FROM - combined_order_data - group by + trade_data_processed_with_prices + GROUP BY solver, tx_hash ), @@ -376,10 +158,10 @@ batch_network_fees AS ( SELECT solver, tx_hash, - sum(network_fee * network_fee_token_native_price) as network_fee + sum(network_fee * network_fee_token_native_price) AS network_fee FROM - combined_order_data - group by + trade_data_processed_with_prices + GROUP BY solver, tx_hash ), @@ -390,7 +172,7 @@ reward_data AS ( ss.auction_id, -- TODO - Assuming that `solver == winner` when both not null -- We will need to monitor that `solver == winner`! - coalesce(os.solver, winner) as solver, + ss.winner as solver, block_number as settlement_block, block_deadline, coalesce(execution_cost, 0) as execution_cost, @@ -403,8 +185,6 @@ reward_data AS ( else 0 end as observed_score, reference_score, - -- auction_participation - participating_solvers, -- protocol_fees coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee, coalesce( @@ -413,13 +193,13 @@ reward_data AS ( ) as network_fee FROM settlement_scores ss - -- If there are reported scores, - -- there will always be a record of auction participants - JOIN auction_participation ap ON ss.auction_id = ap.auction_id -- outer joins made in order to capture non-existent settlements. LEFT OUTER JOIN observed_settlements os ON os.auction_id = ss.auction_id LEFT OUTER JOIN batch_protocol_fees bpf ON bpf.tx_hash = os.tx_hash LEFT OUTER JOIN batch_network_fees bnf ON bnf.tx_hash = os.tx_hash + WHERE + ss.block_deadline > {{start_block}} + AND ss.block_deadline <= {{end_block}} ), reward_per_auction as ( SELECT @@ -442,8 +222,7 @@ reward_per_auction as ( {{EPSILON_UPPER}} ) as capped_payment, winning_score, - reference_score, - participating_solvers as participating_solvers + reference_score FROM reward_data ) @@ -462,8 +241,7 @@ SELECT uncapped_payment :: text as uncapped_payment_eth, capped_payment :: text as capped_payment, winning_score :: text as winning_score, - reference_score :: text as reference_score, - participating_solvers + reference_score :: text as reference_score FROM reward_per_auction ORDER BY block_deadline diff --git a/src/sql/orderbook/prod_order_rewards.sql b/src/sql/orderbook/prod_order_rewards.sql index c18990af..96111de7 100644 --- a/src/sql/orderbook/prod_order_rewards.sql +++ b/src/sql/orderbook/prod_order_rewards.sql @@ -33,293 +33,145 @@ with trade_hashes as ( settlement.block_number > {{start_block}} and settlement.block_number <= {{end_block}} ), -order_surplus AS ( +-- order data +order_data AS ( SELECT - ss.winner as solver, + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM orders + UNION ALL + SELECT + uid, + sell_token, + buy_token, + sell_amount, + buy_amount, + kind, + app_data + FROM jit_orders +), +protocol_fee_kind AS ( + SELECT DISTINCT ON (fp.auction_id, fp.order_uid) + fp.auction_id, + fp.order_uid, + fp.kind + FROM fee_policies fp + JOIN trade_hashes th + ON fp.auction_id = th.auction_id AND fp.order_uid = th.order_uid + ORDER BY fp.auction_id, fp.order_uid, fp.application_order +), +-- unprocessed trade data +trade_data_unprocessed AS ( + SELECT + ss.winner AS solver, s.auction_id, s.tx_hash, t.order_uid, - o.sell_token, - o.buy_token, + od.sell_token, + od.buy_token, t.sell_amount, -- the total amount the user sends t.buy_amount, -- the total amount the user receives - oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price - o.kind, - CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount) - WHEN o.kind = 'buy' THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount - END AS surplus, + oe.surplus_fee AS observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price + od.kind, CASE - WHEN o.kind = 'sell' THEN t.buy_amount - t.sell_amount * (oq.buy_amount - oq.buy_amount / oq.sell_amount * oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.sell_amount - WHEN o.kind = 'buy' THEN t.buy_amount * (oq.sell_amount + oq.gas_amount * oq.gas_price / oq.sell_token_price) / oq.buy_amount - t.sell_amount - END AS price_improvement, - CASE - WHEN o.kind = 'sell' THEN o.buy_token - WHEN o.kind = 'buy' THEN o.sell_token + WHEN od.kind = 'sell' THEN od.buy_token + WHEN od.kind = 'buy' THEN od.sell_token END AS surplus_token, - ad.full_app_data as app_data + convert_from(ad.full_app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' AS partner_fee_recipient, + COALESCE(oe.protocol_fee_amounts[1], 0) AS first_protocol_fee_amount, + COALESCE(oe.protocol_fee_amounts[2], 0) AS second_protocol_fee_amount FROM settlements s JOIN settlement_scores ss -- contains block_deadline ON s.auction_id = ss.auction_id JOIN trades t -- contains traded amounts - ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block - JOIN orders o -- contains tokens and limit amounts - ON t.order_uid = o.uid + ON s.block_number = t.block_number -- given the join that follows with the order execution table, this works even when multiple txs appear in the same block + JOIN order_data od -- contains tokens and limit amounts + ON t.order_uid = od.uid JOIN order_execution oe -- contains surplus fee ON t.order_uid = oe.order_uid AND s.auction_id = oe.auction_id - LEFT OUTER JOIN order_quotes oq -- contains quote amounts - ON o.uid = oq.order_uid LEFT OUTER JOIN app_data ad -- contains full app data - on o.app_data = ad.contract_app_data + ON od.app_data = ad.contract_app_data WHERE - ss.block_deadline >= {{start_block}} - -- since this table filtered on block_deadline is joined with another table filtered on block_number - -- the bound for this table need to be a bit looser. - AND ss.block_deadline <= {{end_block}} + 100 -), -fee_policies_first_proxy as ( - select - auction_id, - order_uid, - max(application_order) as application_order, - count (*) as num_policies - from fee_policies - where auction_id in (select auction_id from order_surplus) - group by order_uid, auction_id -), -fee_policies_first as ( - select - fp.auction_id, - fp.order_uid, - fp.application_order, - fp.kind, - fp.surplus_factor, - fp.surplus_max_volume_factor, - fp.volume_factor, - fp.price_improvement_factor, - fp.price_improvement_max_volume_factor - from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order -), -fee_policies_temp as ( - select - * - from fee_policies - where auction_id in (select auction_id from order_surplus) - except (select * from fee_policies_first ) + s.block_number > {{start_block}} + AND s.block_number <= {{end_block}} ), -fee_policies_second as ( - select - * - from fee_policies_temp - UNION - select - auction_id, - order_uid, - 0 as application_order, - 'volume' as kind, - null as surplus_factor, - null as surplus_max_volume_factor, - 0 as volume_factor, - null as price_improvement_factor, - null as price_improvement_max_volume_factor - from fee_policies_first_proxy where num_policies = 1 -), -order_protocol_fee_first AS ( +-- processed trade data: +trade_data_processed AS ( SELECT - os.auction_id, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.surplus, - os.price_improvement, - os.kind, - convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind_first, + tdu.auction_id, + tdu.solver, + tdu.tx_hash, + tdu.order_uid, + tdu.sell_amount, + tdu.buy_amount, + tdu.sell_token, + tdu.observed_fee, + tdu.surplus_token, + tdu.second_protocol_fee_amount, + tdu.first_protocol_fee_amount + tdu.second_protocol_fee_amount AS protocol_fee, + tdu.partner_fee_recipient, CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * os.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * os.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * os.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * os.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - fp.volume_factor / (1 - fp.volume_factor) * os.buy_amount - WHEN os.kind = 'buy' THEN - fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount - END - END AS protocol_fee_first, - os.surplus_token AS protocol_fee_token + WHEN tdu.partner_fee_recipient IS NOT NULL THEN tdu.second_protocol_fee_amount + ELSE 0 + END AS partner_fee, + tdu.surplus_token AS protocol_fee_token, + pfk.kind as protocol_fee_kind FROM - order_surplus os - JOIN fee_policies_first fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_unprocessed tdu + LEFT OUTER JOIN protocol_fee_kind pfk + ON tdu.order_uid = pfk.order_uid AND tdu.auction_id = pfk.auction_id ), -order_surplus_intermediate as ( - select - auction_id, - order_uid, - CASE - WHEN kind = 'sell' then sell_amount - ELSE sell_amount - protocol_fee_first - END as sell_amount, - CASE - WHEN kind = 'sell' then buy_amount + protocol_fee_first - ELSE buy_amount - END as buy_amount, - surplus + protocol_fee_first as surplus, - price_improvement + protocol_fee_first as price_improvement, - protocol_fee_kind_first, - protocol_fee_first, - partner_fee_recipient - from order_protocol_fee_first -), -order_protocol_fee as ( +price_data AS ( SELECT - os.auction_id, - os.solver, - os.tx_hash, - os.order_uid, - os.sell_amount, - os.buy_amount, - os.sell_token, - os.observed_fee, - os.surplus, - os.surplus_token, - protocol_fee_kind_first, - fp.kind as protocol_fee_kind_second, - protocol_fee_first, - CASE - WHEN fp.kind = 'surplus' THEN CASE - WHEN os.kind = 'sell' THEN - -- We assume that the case surplus_factor != 1 always. In - -- that case reconstructing the protocol fee would be - -- impossible anyways. This query will return a division by - -- zero error in that case. - protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, - -- at most charge a fraction of volume - fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus - ) - END - WHEN fp.kind = 'priceimprovement' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, - -- charge a fraction of price improvement, at most 0 - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement - , - 0 - ) - ) - WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( - -- at most charge a fraction of volume - fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, - -- charge a fraction of price improvement - GREATEST( - fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, - 0 - ) - ) - END - WHEN fp.kind = 'volume' THEN CASE - WHEN os.kind = 'sell' THEN - protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount - WHEN os.kind = 'buy' THEN - protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount - END - END AS protocol_fee, - osi.partner_fee_recipient, - CASE - WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first - ELSE 0 - END AS partner_fee, - os.surplus_token AS protocol_fee_token + tdp.auction_id, + tdp.order_uid, + ap_surplus.price / pow(10, 18) AS surplus_token_native_price, + ap_protocol.price / pow(10, 18) AS protocol_fee_token_native_price, + ap_sell.price / pow(10, 18) AS network_fee_token_native_price FROM - order_surplus os - JOIN order_surplus_intermediate osi - ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id - JOIN fee_policies_second fp -- contains protocol fee policy - ON os.auction_id = fp.auction_id - AND os.order_uid = fp.order_uid + trade_data_processed AS tdp + LEFT OUTER JOIN auction_prices ap_sell -- contains price: sell token + ON tdp.auction_id = ap_sell.auction_id + AND tdp.sell_token = ap_sell.token + LEFT OUTER JOIN auction_prices ap_surplus -- contains price: surplus token + ON tdp.auction_id = ap_surplus.auction_id + AND tdp.surplus_token = ap_surplus.token + LEFT OUTER JOIN auction_prices ap_protocol -- contains price: protocol fee token + ON tdp.auction_id = ap_protocol.auction_id + AND tdp.surplus_token = ap_protocol.token ), -order_protocol_fee_prices AS ( +trade_data_processed_with_prices AS ( SELECT - opf.auction_id, - opf.solver, - opf.tx_hash, - opf.order_uid, - opf.surplus, - opf.protocol_fee, - opf.protocol_fee_token, - CASE - WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second - ELSE opf.protocol_fee_kind_first - END AS protocol_fee_kind, - opf.partner_fee, - opf.partner_fee_recipient, + tdp.auction_id, + tdp.solver, + tdp.tx_hash, + tdp.order_uid, + tdp.surplus_token, + tdp.protocol_fee, + tdp.protocol_fee_token, + tdp.partner_fee, + tdp.partner_fee_recipient, CASE - WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee - ELSE opf.protocol_fee - END AS network_fee_correction, - opf.sell_token as network_fee_token, - ap_surplus.price / pow(10, 18) as surplus_token_native_price, - ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, - ap_sell.price / pow(10, 18) as network_fee_token_native_price + WHEN tdp.sell_token != tdp.surplus_token THEN tdp.observed_fee - (tdp.sell_amount - tdp.observed_fee) / tdp.buy_amount * COALESCE(tdp.protocol_fee, 0) + ELSE tdp.observed_fee - COALESCE(tdp.protocol_fee, 0) + END AS network_fee, + tdp.sell_token AS network_fee_token, + surplus_token_native_price, + protocol_fee_token_native_price, + network_fee_token_native_price, + protocol_fee_kind FROM - order_protocol_fee as opf - JOIN auction_prices ap_sell -- contains price: sell token - ON opf.auction_id = ap_sell.auction_id - AND opf.sell_token = ap_sell.token - JOIN auction_prices ap_surplus -- contains price: surplus token - ON opf.auction_id = ap_surplus.auction_id - AND opf.surplus_token = ap_surplus.token - JOIN auction_prices ap_protocol -- contains price: protocol fee token - ON opf.auction_id = ap_protocol.auction_id - AND opf.protocol_fee_token = ap_protocol.token + trade_data_processed AS tdp + JOIN price_data pd + ON tdp.auction_id = pd.auction_id + AND tdp.order_uid = pd.order_uid ), winning_quotes as ( SELECT @@ -365,14 +217,15 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - cast(coalesce(opfp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, - opfp.partner_fee_recipient, - opfp.protocol_fee_kind + cast(coalesce(tdpwp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, + tdpwp.partner_fee_recipient, + tdpwp.protocol_fee_kind from trade_hashes left outer join order_execution o on trade_hashes.order_uid = o.order_uid and trade_hashes.auction_id = o.auction_id left outer join winning_quotes wq on trade_hashes.order_uid = wq.order_uid - left outer join order_protocol_fee_prices opfp on trade_hashes.order_uid = opfp.order_uid - and trade_hashes.auction_id = opfp.auction_id + left outer join trade_data_processed_with_prices tdpwp on trade_hashes.order_uid = tdpwp.order_uid + and trade_hashes.auction_id = tdpwp.auction_id left outer join order_quotes oq on trade_hashes.order_uid = oq.order_uid + order by trade_hashes.block_number asc diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 928abf07..523d0aca 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -15,64 +15,64 @@ def test_latest_block_increasing(self): self.assertGreaterEqual(OrderbookFetcher.get_latest_block(), latest_block) def test_get_order_rewards(self): - block_number = 19184750 - block_range = BlockRange(block_number, block_number + 25) + block_number = 20934809 + block_range = BlockRange(block_number, block_number + 17) rewards_df = OrderbookFetcher.get_order_rewards(block_range) expected = pd.DataFrame( { "block_number": [ - 19184754, - 19184754, - 19184754, - 19184754, - 19184754, - 19184754, - 19184763, + 20934810, + 20934812, + 20934816, + 20934819, + 20934821, + 20934823, + 20934823, ], "order_uid": [ - "0x12228107308040cd64293415c7338b7ec752338aad9738deb9d10bed27cbf81558203dd7b18b262f7a422f75d79cbb230752a74e65c50d9a", - "0x05c68d2a038f4feb748fbc7622a3bfb6d2484c56d616e3e3b265d4ad52d177ee40a50cf069e992aa4536211b23f286ef88752187ffffffff", - "0x02bffb58877c193873693164a077c7298f470088bc517a77b6eb7d882d5d267e6ce4e9d75a1d6340812284ef564c256061cf10ba65c50cc1", - "0x85d3fabb73d7029e0b3eceeeb1fe46a859a5a2aa52a482511950ba633921e63f911da057b7f735bc54e9ffbe1143dda363dba6fb65c50ce2", - "0x0d44fb9adc79e104645ddcc4e04b36953e6a37a6da0c2c145350425a09ff9132851064900dcb11351e53a39e5af340e9fc2b768965c50d54", - "0xa5d8c049983f2170a37180d01055b92ff67c0db2ccb9cf5d4ec7ee4c095d157f40a50cf069e992aa4536211b23f286ef88752187ffffffff", - "0x95e26eba547575e86a81e6e7be87b45b56f2fd824b4bcc573bfb3f7a44b2c3c340a50cf069e992aa4536211b23f286ef88752187ffffffff", + "0xc129dbf2f44b67a570a161860de24074a739c6d95d23eee736149f6ea55d84ac36eb8d32526284fa6f2dd151b982e52988d7b5f86707bc6b", + "0xe9464aea08101cd1c5ecb6e896554b502e7e9759e9f7159189311790aae698c28a1f2c41f6f96730623d860c45ec01a8a63b3ff86707bc95", + "0x1579de80933bfe2397a685d246e309ea4a9b8696597b8b957cabe0921f3eb8b9dc1e9a21b7ed8dfb87c997e7d0405e2cfcfa89386707bcca", + "0x8eb73ae608594d507a5cfb0d90088d6cf2ec431dca46505a460fbd3dbaf07acf04fbb30283c28fa7e989ed5c91a2fc338973f34e6707b7fe", + "0xf933af0bcf500b5499d9802f01f85dd13d0cde0e48aa08cefadb738438a2b06540a50cf069e992aa4536211b23f286ef88752187ffffffff", + "0xe04adff20d737ca106ba70856bf7e0cc1800d5de1783a5dbc49c555e3fc6463940a50cf069e992aa4536211b23f286ef88752187ffffffff", + "0xb0b69286396dd53acb74e9fb25d6a9a72dd81189e3d301140c9f0cb5f2fb988b3e4aea5c2e1433910621b4378dfa541264fc145b6707bcfb", ], "solver": [ "0x4339889fd9dfca20a423fba011e9dff1c856caeb", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", + "0xc7899ff6a3ac2ff59261bd960a8c880df06e1041", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", - "0x6f799f4bf6c1c56fb8d890e9e0fff2934b0de157", ], "quote_solver": [ + "0x008300082c3000009e63680088f8c7f4d3ff2e87", + "0x008300082c3000009e63680088f8c7f4d3ff2e87", "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0x16c473448e770ff647c69cbe19e28528877fba1b", - None, - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0x4339889fd9dfca20a423fba011e9dff1c856caeb", + "0xc10d4dfda62227d9ec23ab0010e2942e48338a60", "0x4339889fd9dfca20a423fba011e9dff1c856caeb", + "0xc7899ff6a3ac2ff59261bd960a8c880df06e1041", + "0x008300082c3000009e63680088f8c7f4d3ff2e87", ], "tx_hash": [ - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x43699f7356df831aa04519ecd9f638303f716ab9e41c6ab1f38aacd1c9cdd3e5", - "0x321703689c7a2743c85ad91788b8ab00447de42591089d2052a3e9cf3be9d5a1", + "0x54cd9572979c952f2e1b0a966e42ad004e243367f6349e2ce56d80e91db2978a", + "0x425b0d49dfdf860c08d21da224f20a2bb7606c8030ddb5b21b4cd98a70502740", + "0x84cf5e44ce00d82105702d7f45e20a9fb60a479e494262ccf621b79f7770a490", + "0x3483aaa340e524077f857d3d94e05a551839cc835909ac81450e21f22c902a79", + "0x96318915e712ea346fa99cbe58519db6767bc45aad6e1fcc92f48f5a02b8e034", + "0xba7475c759c289e85ee0dd3319e39e405d26e3066f457df50c9110a64e601e6b", + "0xba7475c759c289e85ee0dd3319e39e405d26e3066f457df50c9110a64e601e6b", ], "surplus_fee": [ - "0", - "0", - "57709403", - "0", - "0", - "0", - "0", + "2266472706677075", + "1189015100233555732594687", + "2837993604", + "5512297385347536", + "2302819091231050", + "4797029976282923", + "407690373176599", ], "amount": [ 0.0, @@ -86,65 +86,65 @@ def test_get_order_rewards(self): "protocol_fee": [ "0", "0", - "4382713635683340000", - "0", - "0", - "0", + "467895382571196", + "2286532007592231309", + "17883214117504888", "0", + "297494968688870", ], "protocol_fee_token": [ - None, - None, - "0xd9fcd98c322942075a5c3860693e9f4f03aae07b", - None, - None, - None, - None, + "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "0xf19308f923582a6f7c465e5ce7a9dc1bec6665b1", + "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "0xec53bf9167f50cdeb3ae105f56099aaab9061f83", + "0x3106a0a076bedae847652f42ef07fd58589e001f", + "0xfd0205066521550d7d7ab19da8f72bb004b4c341", + "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", ], "protocol_fee_native_price": [ - 0.0, - 0.0, - 0.001552064411669457, - 0.0, - 0.0, - 0.0, - 0.0, + 1.0, + 4.14627277e-10, + 1.0, + 0.001495472574158779, + 0.00008227599141422, + 0.000012405155005943, + 1.0, ], "quote_sell_amount": [ - "4429378850", - "184573231326057476", - "7500000000", - "192465488420922607", - "1954467549", - "856446990599561516", - "71274765128486370400", + "109899794590184336", + "6678762644284597849428691687", + "8000000000000", + "5073337115967525579", + "1469072907309463969", + "6000000000000000000", + "29670143598685664", ], "quote_buy_amount": [ - "12009545997505395163", - "143919269475547068", - "1920099071497969074176", - "110185117837140289", - "427611951239582233762", - "1465179606977149402", - "69200861384935994787", + "110611848308261632", + "22567439059361636779445714944", + "8849796343220294600", + "3384931672358364844032", + "17598530665993045803008", + "476716533757541695910447", + "29862362965928048", ], "quote_gas_cost": [ - 28980049962721130.0, - 15426768673942524.0, - 18282786590639504.0, - 18696459349125884.0, - 18629380517476576.0, - 23553009400438484.0, - 25234871513629598.0, + 2195335897623140.2, + 1497753531187998.5, + 3278739441054120.0, + 2398711536964080.0, + 2053290592851789.0, + 3228942613973600.0, + 2097897905068290.0, ], "quote_sell_token_price": [ - 410359364.5808475, - 1.0, - 408619711.8704729, - 0.538163462286757, - 409145130.1194029, - 1.0, - 1.0, + 1.0073075605014072, + 1.3148021574880433e-9, + 1111228.9670799475, + 1, + 1, + 1, + 1.0073055030210916, ], "partner_fee": [ "0", @@ -165,13 +165,13 @@ def test_get_order_rewards(self): None, ], "protocol_fee_kind": [ - None, - None, - "surplus", - None, - None, - None, - None, + "priceimprovement", + "priceimprovement", + "priceimprovement", + "priceimprovement", + "priceimprovement", + "priceimprovement", + "priceimprovement", ], } ) @@ -179,243 +179,168 @@ def test_get_order_rewards(self): self.assertIsNone(pd.testing.assert_frame_equal(expected, rewards_df)) def test_get_batch_rewards(self): - block_number = 19468910 - block_range = BlockRange(block_number, block_number + 25) + block_number = 20936269 + block_range = BlockRange(block_number, block_number + 11) rewards_df = OrderbookFetcher.get_batch_rewards(block_range) expected = pd.DataFrame( { "block_number": pd.Series( - [19468907, 19468912, pd.NA, pd.NA, 19468922, 19468924, 19468926], + [20936268, 20936270, pd.NA, 20936274, pd.NA, 20936278], dtype="Int64", ), "block_deadline": [ - 19468915, - 19468919, - 19468927, - 19468928, - 19468930, - 19468933, - 19468935, + 20936270, + 20936272, + 20936274, + 20936276, + 20936278, + 20936280, ], "tx_hash": [ - "0x9b46f8197480f452bf88b928706e6a6d5c0d0fc1f90478b9869d06be8f972602", - "0x2f045a84e3acef4680aa8d27c48b4f8c7639857e6db9333edf627166c7eeb542", + "0x623a8b23d2eca21f2661636b814725ac002f11b1174dbe38bdd5d2fb0e91369d", + "0xd099e924132c2c3be2ded1f9dae0f0910025b97f5b639260cafb8775038c60dc", None, + "0x147119b2fcc44b8f8a34e733edf0c5fbc182e70845da1eb408f6d5b7f0722006", None, - "0x2702518fba3abac804863e64800c26615d7d14020bd0c6f9e17d98151e987354", - "0xeb4bc0d934c1b87644a52a21e182edda559a44e91c5afe6a08ec4e1fba21a626", - "0xe8f5a56521801c0978c203654940ae6dfd26e414e7018749a0456965168e8e9f", + "0x6f133a1790397020aad2ac2e18496448c08c2a0bea2ae6e3667f3e7e2b67ffd0", ], "solver": [ "0x9b7e7f21d98f21c0354035798c40e9040e25787f", - "0xbf54079c9bc879ae4dd6bc79bce11d3988fd9c2b", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0x9b7e7f21d98f21c0354035798c40e9040e25787f", + "0x4339889fd9dfca20a423fba011e9dff1c856caeb", + "0x008300082c3000009e63680088f8c7f4d3ff2e87", + "0x0ddcb0769a3591230caa80f85469240b71442089", + "0x008300082c3000009e63680088f8c7f4d3ff2e87", + "0xc7899ff6a3ac2ff59261bd960a8c880df06e1041", ], "execution_cost": [ - "9740476311332981", - "18230688793672800", + "12983200929406728", + "15957284641911096", "0", + "8042014217798700", "0", - "17029862092746496", - "14696413493550464", - "14816441369559104", + "10406353405972980", ], "surplus": [ - "11327896430820094", - "33419292238225043", + "84730642639483901", + "447938889765280735", "0", + "1962451333659395", "0", - "14062656113719218", - "9557249513545248", - "5267492734979407", + "114169776361571642", ], "protocol_fee": [ - "58945874579", - "0", - "0", "0", + "71690869135785700", "0", "0", "0", + "26385186071416400", ], "network_fee": [ - "7715157183718640", - "18639716210784800", + "9477057450023950", + "16474626289268700", "0", + "7125533240475470", "0", - "15011842152833800", - "21801311054855300", - "8912931423922370", + "7169186891203840", ], "uncapped_payment_eth": [ - "405658366132340", - "292618683544035", - "-13360904627424245", - "-12938581202699558", - "2823075137791897", - "9345865552057182", - "1607086130097915", + "4049209356932509", + "21185897680263397", + "-370478108035088499", + "519803564128039", + "-357155743729567764", + "2367258180235328", ], "capped_payment": [ - "405658366132340", - "292618683544035", + "4049209356932509", + "12000000000000000", "-10000000000000000", + "519803564128039", "-10000000000000000", - "2823075137791897", - "9345865552057182", - "1607086130097915", + "2367258180235328", ], "winning_score": [ - "11327955072945657", - "33419292238225043", - "15862581044125306", - "14047809923397763", - "14062656113719218", - "9557249513545248", - "5267492734979406", + "84730642639483901", + "519629758901066469", + "387260436996575552", + "1962451333659395", + "374093887861962368", + "140554962432988068", ], "reference_score": [ - "10922296706813317", - "33126673554681008", - "13360904627424245", - "12938581202699558", - "11239580975927321", - "211383961488066", - "3660406604881491", - ], - "participating_solvers": [ - [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0x4fc4a61a3b99a1ad4a61b03f3752ca12b4a17646", - "0x9b7e7f21d98f21c0354035798c40e9040e25787f", - "0xb9332b6301e5983272c30dd5b48a4e3b1664511b", - "0xbf54079c9bc879ae4dd6bc79bce11d3988fd9c2b", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - "0xd50ecb485dcf5d97266122dfed979587dd8923ac", - ], - [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0xbf54079c9bc879ae4dd6bc79bce11d3988fd9c2b", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], - [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], - [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], - [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], - [ - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xc9f2e6ea1637e499406986ac50ddc92401ce1f58", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], - [ - "0x0ddcb0769a3591230caa80f85469240b71442089", - "0x4339889fd9dfca20a423fba011e9dff1c856caeb", - "0x4fc4a61a3b99a1ad4a61b03f3752ca12b4a17646", - "0x9b7e7f21d98f21c0354035798c40e9040e25787f", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", - "0xd1508a211d98bb81195dc1f9533edcdf68adf036", - ], + "80681433282551392", + "498443861220803072", + "370478108035088499", + "1442647769531356", + "357155743729567764", + "138187704252752740", ], }, ) self.assertIsNone(pd.testing.assert_frame_equal(expected, rewards_df)) def test_get_order_rewards_with_integrator_fee(self): - block_range = BlockRange(19581572, 19581581) + block_range = BlockRange(20934805, 20934809) rewards_df = OrderbookFetcher.get_order_rewards(block_range) expected = pd.DataFrame( { "block_number": [ - 19581573, - 19581573, - 19581579, + 20934806, + 20934808, ], "order_uid": [ - "0x0f6c83ff144aabed918417f61a92672165bba9b1c90f078fedfc10c2c16d03d09fa3c00a92ec5f96b1ad2527ab41b3932efeda58660e7959", - "0xac38b37c7821b1c0f0fd631912d7b19581c305c4bdca84dcb6d862da6cf8591cf9d7adfb5bc41283d67db9d71add65162e242f62660e7c82", - "0xc9c870decab00c1335babef5a3009186671b1bc69131c7338a4ecb2ed14831c321a01f63b263d93e124471e456578024121a6b78660e7a5b", + "0x3d388aacd736a1043ca403b0ecfd9cd4fd1a46cebab99f74637d7d1bfd08200631d7916593d1ed6cdb3b1ff2a1f656a2eb2a23816707bbfd", + "0xad7064d19ab9398fc4a0f17e57bea8d4ed83a71d91c1798e3da01199540b970eecf1c70100107017c5da1ff4bbcc482809e396ee6707ba81", ], "solver": [ - "0x4339889fd9dfca20a423fba011e9dff1c856caeb", - "0x4339889fd9dfca20a423fba011e9dff1c856caeb", - "0xc74b656bd2ebe313d26d1ac02bcf95b137d1c857", + "0x755bae1cd46c9c27a3230aef0ce923bda13d29f7", + "0xd1508a211d98bb81195dc1f9533edcdf68adf036", ], "quote_solver": [ - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0x16c473448e770ff647c69cbe19e28528877fba1b", - "0x16c473448e770ff647c69cbe19e28528877fba1b", + "0x4fc4a61a3b99a1ad4a61b03f3752ca12b4a17646", + "0xd1508a211d98bb81195dc1f9533edcdf68adf036", ], "tx_hash": [ - "0x13315e833ed3204db3320e3e8d213c84ab21e55e715847514d78198af4f68861", - "0x13315e833ed3204db3320e3e8d213c84ab21e55e715847514d78198af4f68861", - "0x962af6dbd9940249b8a795b48c0c4cbc04c0444555bb5b16fd4d824261e282bf", + "0x08f51e088100f4afddf4012e7b6b26d292861ceda92723fd764fff3e1af49c0d", + "0x6ea58eb10d57134c0fb0701c208292eade107870029d0653404a2371d63588fa", ], "surplus_fee": [ - "958869097252287", - "29812970679943659325", - "31963685928336242096", + "7606", + "144778705", ], - "amount": [0.0, 0.0, 0.0], - "protocol_fee": ["346011", "0", "0"], + "amount": [0.0, 0.0], + "protocol_fee": ["1329681", "140147111"], "protocol_fee_token": [ - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", - "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", + "0xdac17f958d2ee523a2206206994597c13d831ec7", + "0xdac17f958d2ee523a2206206994597c13d831ec7", ], "protocol_fee_native_price": [ - 299379646.7519555, - 299379646.7519555, - 299379646.7519555, + 417562067.399692, + 417562067.399692, ], "quote_sell_amount": [ - "11177061073424246", - "30000000000000000000000", - "667000000000000000000", + "2500000", + "200000000000", ], - "quote_buy_amount": ["37147277", "22954359609", "358516953"], + "quote_buy_amount": ["1523804298", "200214933429"], "quote_gas_cost": [ - 3605211063076992.0, - 1982659477433822.0, - 2865645675046151.0, + 1808592978505935, + 1516678038740264.2, ], "quote_sell_token_price": [ - 1.0, - 0.000234609143374563, - 0.000160802298220274, + 254238788483.4054, + 417824591.695399, ], "partner_fee": [ - "346011", - "0", "0", + "140147111", ], "partner_fee_recipient": [ - "0x9FA3c00a92Ec5f96B1Ad2527ab41B3932EFEDa58", - None, None, + "0x63695Eee2c3141BDE314C5a6f89B98E62808d716", ], "protocol_fee_kind": [ - "volume", "priceimprovement", "priceimprovement", ],