From e2c4db920d3e2d607f3a2bbe49df72428982d877 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Wed, 26 Jun 2024 14:33:23 +0200 Subject: [PATCH 1/2] fix multiple protocol fees and upload partner fee --- src/models/order_rewards_schema.py | 2 + src/sql/orderbook/barn_batch_rewards.sql | 169 +++++++++++++++++++++-- src/sql/orderbook/barn_order_rewards.sql | 166 ++++++++++++++++++++-- src/sql/orderbook/prod_batch_rewards.sql | 169 +++++++++++++++++++++-- src/sql/orderbook/prod_order_rewards.sql | 166 ++++++++++++++++++++-- tests/unit/test_order_rewards_schema.py | 4 + 6 files changed, 634 insertions(+), 42 deletions(-) diff --git a/src/models/order_rewards_schema.py b/src/models/order_rewards_schema.py index 79b87392..f9543461 100644 --- a/src/models/order_rewards_schema.py +++ b/src/models/order_rewards_schema.py @@ -1,4 +1,5 @@ """Model for Order Rewards Data""" + from __future__ import annotations from dataclasses import dataclass @@ -35,6 +36,7 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "quote_buy_amount": str(row["quote_buy_amount"]), "quote_gas_cost": float(row["quote_gas_cost"]), "quote_sell_token_price": float(row["quote_sell_token_price"]), + "partner_fee": str(row["partner_fee"]), "partner_fee_recipient": row["partner_fee_recipient"], "protocol_fee_kind": row["protocol_fee_kind"], }, diff --git a/src/sql/orderbook/barn_batch_rewards.sql b/src/sql/orderbook/barn_batch_rewards.sql index 27d22889..b93cdae2 100644 --- a/src/sql/orderbook/barn_batch_rewards.sql +++ b/src/sql/orderbook/barn_batch_rewards.sql @@ -59,7 +59,8 @@ order_surplus AS ( CASE WHEN o.kind = 'sell' THEN o.buy_token WHEN o.kind = 'buy' THEN o.sell_token - END AS surplus_token + END AS surplus_token, + ad.full_app_data as app_data FROM settlements s JOIN settlement_scores ss -- contains block_deadline @@ -73,22 +74,70 @@ order_surplus AS ( AND s.auction_id = oe.auction_id LEFT OUTER JOIN order_quotes oq -- contains quote amounts ON o.uid = oq.order_uid + LEFT OUTER JOIN app_data ad -- contains full app data + on o.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), -order_protocol_fee AS ( +fee_policies_first_proxy as ( + select + auction_id, + order_uid, + max(application_order) as application_order, + count (*) as num_policies + from fee_policies + where auction_id in (select auction_id from order_surplus) + group by order_uid, auction_id +), +fee_policies_first as ( + select + fp.auction_id, + fp.order_uid, + fp.application_order, + fp.kind, + fp.surplus_factor, + fp.surplus_max_volume_factor, + fp.volume_factor, + fp.price_improvement_factor, + fp.price_improvement_max_volume_factor + from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order +), +fee_policies_temp as ( + select + * + from fee_policies + where auction_id in (select auction_id from order_surplus) + except (select * from fee_policies_first ) +), +fee_policies_second as ( + select + * + from fee_policies_temp + UNION + select + auction_id, + order_uid, + 0 as application_order, + 'volume' as kind, + null as surplus_factor, + null as surplus_max_volume_factor, + 0 as volume_factor, + null as price_improvement_factor, + null as price_improvement_max_volume_factor + from fee_policies_first_proxy where num_policies = 1 +), +order_protocol_fee_first AS ( SELECT os.auction_id, - os.solver, - os.tx_hash, os.order_uid, os.sell_amount, os.buy_amount, - os.sell_token, - os.observed_fee, os.surplus, - os.surplus_token, + os.price_improvement, + os.kind, + convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, + fp.kind as protocol_fee_kind_first, CASE WHEN fp.kind = 'surplus' THEN CASE WHEN os.kind = 'sell' THEN @@ -135,11 +184,106 @@ order_protocol_fee AS ( WHEN os.kind = 'buy' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount END + END AS protocol_fee_first, + os.surplus_token AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies_first fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_surplus_intermediate as ( + select + auction_id, + order_uid, + CASE + WHEN kind = 'sell' then sell_amount + ELSE sell_amount - protocol_fee_first + END as sell_amount, + CASE + WHEN kind = 'sell' then buy_amount + protocol_fee_first + ELSE buy_amount + END as buy_amount, + surplus + protocol_fee_first as surplus, + price_improvement + protocol_fee_first as price_improvement, + protocol_fee_kind_first, + protocol_fee_first, + partner_fee_recipient + from order_protocol_fee_first +), +order_protocol_fee as ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + protocol_fee_kind_first, + fp.kind as protocol_fee_kind_second, + protocol_fee_first, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'priceimprovement' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, + -- charge a fraction of price improvement, at most 0 + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement + , + 0 + ) + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, + -- charge a fraction of price improvement + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, + 0 + ) + ) + END + WHEN fp.kind = 'volume' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount + WHEN os.kind = 'buy' THEN + protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount + END END AS protocol_fee, + osi.partner_fee_recipient, + CASE + WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + ELSE 0 + END AS partner_fee, os.surplus_token AS protocol_fee_token FROM order_surplus os - JOIN fee_policies fp -- contains protocol fee policy + JOIN order_surplus_intermediate osi + ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id + JOIN fee_policies_second fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), @@ -152,6 +296,12 @@ order_protocol_fee_prices AS ( opf.surplus, opf.protocol_fee, opf.protocol_fee_token, + CASE + WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second + ELSE opf.protocol_fee_kind_first + END AS protocol_fee_kind, + opf.partner_fee, + opf.partner_fee_recipient, CASE WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee ELSE opf.protocol_fee @@ -161,7 +311,7 @@ order_protocol_fee_prices AS ( ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, ap_sell.price / pow(10, 18) as network_fee_token_native_price FROM - order_protocol_fee opf + order_protocol_fee as opf JOIN auction_prices ap_sell -- contains price: sell token ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token @@ -176,7 +326,6 @@ batch_protocol_fees AS ( SELECT solver, tx_hash, - -- sum(surplus * surplus_token_price) as surplus, sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee, sum(network_fee_correction * network_fee_token_native_price) as network_fee_correction FROM diff --git a/src/sql/orderbook/barn_order_rewards.sql b/src/sql/orderbook/barn_order_rewards.sql index 229ea5ae..21bdd31e 100644 --- a/src/sql/orderbook/barn_order_rewards.sql +++ b/src/sql/orderbook/barn_order_rewards.sql @@ -79,20 +79,64 @@ order_surplus AS ( -- the bound for this table need to be a bit looser. AND ss.block_deadline <= {{end_block}} + 100 ), -order_protocol_fee AS ( +fee_policies_first_proxy as ( + select + auction_id, + order_uid, + max(application_order) as application_order, + count (*) as num_policies + from fee_policies + where auction_id in (select auction_id from order_surplus) + group by order_uid, auction_id +), +fee_policies_first as ( + select + fp.auction_id, + fp.order_uid, + fp.application_order, + fp.kind, + fp.surplus_factor, + fp.surplus_max_volume_factor, + fp.volume_factor, + fp.price_improvement_factor, + fp.price_improvement_max_volume_factor + from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order +), +fee_policies_temp as ( + select + * + from fee_policies + where auction_id in (select auction_id from order_surplus) + except (select * from fee_policies_first ) +), +fee_policies_second as ( + select + * + from fee_policies_temp + UNION + select + auction_id, + order_uid, + 0 as application_order, + 'volume' as kind, + null as surplus_factor, + null as surplus_max_volume_factor, + 0 as volume_factor, + null as price_improvement_factor, + null as price_improvement_max_volume_factor + from fee_policies_first_proxy where num_policies = 1 +), +order_protocol_fee_first AS ( SELECT os.auction_id, - os.solver, - os.tx_hash, os.order_uid, os.sell_amount, os.buy_amount, - os.sell_token, - os.observed_fee, os.surplus, - os.surplus_token, + os.price_improvement, + os.kind, convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind, + fp.kind as protocol_fee_kind_first, CASE WHEN fp.kind = 'surplus' THEN CASE WHEN os.kind = 'sell' THEN @@ -139,11 +183,106 @@ order_protocol_fee AS ( WHEN os.kind = 'buy' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount END + END AS protocol_fee_first, + os.surplus_token AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies_first fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_surplus_intermediate as ( + select + auction_id, + order_uid, + CASE + WHEN kind = 'sell' then sell_amount + ELSE sell_amount - protocol_fee_first + END as sell_amount, + CASE + WHEN kind = 'sell' then buy_amount + protocol_fee_first + ELSE buy_amount + END as buy_amount, + surplus + protocol_fee_first as surplus, + price_improvement + protocol_fee_first as price_improvement, + protocol_fee_kind_first, + protocol_fee_first, + partner_fee_recipient + from order_protocol_fee_first +), +order_protocol_fee as ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + protocol_fee_kind_first, + fp.kind as protocol_fee_kind_second, + protocol_fee_first, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'priceimprovement' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, + -- charge a fraction of price improvement, at most 0 + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement + , + 0 + ) + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, + -- charge a fraction of price improvement + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, + 0 + ) + ) + END + WHEN fp.kind = 'volume' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount + WHEN os.kind = 'buy' THEN + protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount + END END AS protocol_fee, + osi.partner_fee_recipient, + CASE + WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + ELSE 0 + END AS partner_fee, os.surplus_token AS protocol_fee_token FROM order_surplus os - JOIN fee_policies fp -- contains protocol fee policy + JOIN order_surplus_intermediate osi + ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id + JOIN fee_policies_second fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), @@ -156,8 +295,12 @@ order_protocol_fee_prices AS ( opf.surplus, opf.protocol_fee, opf.protocol_fee_token, + CASE + WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second + ELSE opf.protocol_fee_kind_first + END AS protocol_fee_kind, + opf.partner_fee, opf.partner_fee_recipient, - opf.protocol_fee_kind, CASE WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee ELSE opf.protocol_fee @@ -167,7 +310,7 @@ order_protocol_fee_prices AS ( ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, ap_sell.price / pow(10, 18) as network_fee_token_native_price FROM - order_protocol_fee opf + order_protocol_fee as opf JOIN auction_prices ap_sell -- contains price: sell token ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token @@ -222,7 +365,8 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - opfp.partner_fee_recipient as partner_fee_recipient, + opfp.partner_fee, + opfp.partner_fee_recipient, opfp.protocol_fee_kind from trade_hashes diff --git a/src/sql/orderbook/prod_batch_rewards.sql b/src/sql/orderbook/prod_batch_rewards.sql index 27d22889..b93cdae2 100644 --- a/src/sql/orderbook/prod_batch_rewards.sql +++ b/src/sql/orderbook/prod_batch_rewards.sql @@ -59,7 +59,8 @@ order_surplus AS ( CASE WHEN o.kind = 'sell' THEN o.buy_token WHEN o.kind = 'buy' THEN o.sell_token - END AS surplus_token + END AS surplus_token, + ad.full_app_data as app_data FROM settlements s JOIN settlement_scores ss -- contains block_deadline @@ -73,22 +74,70 @@ order_surplus AS ( AND s.auction_id = oe.auction_id LEFT OUTER JOIN order_quotes oq -- contains quote amounts ON o.uid = oq.order_uid + LEFT OUTER JOIN app_data ad -- contains full app data + on o.app_data = ad.contract_app_data WHERE ss.block_deadline > {{start_block}} AND ss.block_deadline <= {{end_block}} ), -order_protocol_fee AS ( +fee_policies_first_proxy as ( + select + auction_id, + order_uid, + max(application_order) as application_order, + count (*) as num_policies + from fee_policies + where auction_id in (select auction_id from order_surplus) + group by order_uid, auction_id +), +fee_policies_first as ( + select + fp.auction_id, + fp.order_uid, + fp.application_order, + fp.kind, + fp.surplus_factor, + fp.surplus_max_volume_factor, + fp.volume_factor, + fp.price_improvement_factor, + fp.price_improvement_max_volume_factor + from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order +), +fee_policies_temp as ( + select + * + from fee_policies + where auction_id in (select auction_id from order_surplus) + except (select * from fee_policies_first ) +), +fee_policies_second as ( + select + * + from fee_policies_temp + UNION + select + auction_id, + order_uid, + 0 as application_order, + 'volume' as kind, + null as surplus_factor, + null as surplus_max_volume_factor, + 0 as volume_factor, + null as price_improvement_factor, + null as price_improvement_max_volume_factor + from fee_policies_first_proxy where num_policies = 1 +), +order_protocol_fee_first AS ( SELECT os.auction_id, - os.solver, - os.tx_hash, os.order_uid, os.sell_amount, os.buy_amount, - os.sell_token, - os.observed_fee, os.surplus, - os.surplus_token, + os.price_improvement, + os.kind, + convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, + fp.kind as protocol_fee_kind_first, CASE WHEN fp.kind = 'surplus' THEN CASE WHEN os.kind = 'sell' THEN @@ -135,11 +184,106 @@ order_protocol_fee AS ( WHEN os.kind = 'buy' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount END + END AS protocol_fee_first, + os.surplus_token AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies_first fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_surplus_intermediate as ( + select + auction_id, + order_uid, + CASE + WHEN kind = 'sell' then sell_amount + ELSE sell_amount - protocol_fee_first + END as sell_amount, + CASE + WHEN kind = 'sell' then buy_amount + protocol_fee_first + ELSE buy_amount + END as buy_amount, + surplus + protocol_fee_first as surplus, + price_improvement + protocol_fee_first as price_improvement, + protocol_fee_kind_first, + protocol_fee_first, + partner_fee_recipient + from order_protocol_fee_first +), +order_protocol_fee as ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + protocol_fee_kind_first, + fp.kind as protocol_fee_kind_second, + protocol_fee_first, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'priceimprovement' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, + -- charge a fraction of price improvement, at most 0 + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement + , + 0 + ) + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, + -- charge a fraction of price improvement + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, + 0 + ) + ) + END + WHEN fp.kind = 'volume' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount + WHEN os.kind = 'buy' THEN + protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount + END END AS protocol_fee, + osi.partner_fee_recipient, + CASE + WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + ELSE 0 + END AS partner_fee, os.surplus_token AS protocol_fee_token FROM order_surplus os - JOIN fee_policies fp -- contains protocol fee policy + JOIN order_surplus_intermediate osi + ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id + JOIN fee_policies_second fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), @@ -152,6 +296,12 @@ order_protocol_fee_prices AS ( opf.surplus, opf.protocol_fee, opf.protocol_fee_token, + CASE + WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second + ELSE opf.protocol_fee_kind_first + END AS protocol_fee_kind, + opf.partner_fee, + opf.partner_fee_recipient, CASE WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee ELSE opf.protocol_fee @@ -161,7 +311,7 @@ order_protocol_fee_prices AS ( ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, ap_sell.price / pow(10, 18) as network_fee_token_native_price FROM - order_protocol_fee opf + order_protocol_fee as opf JOIN auction_prices ap_sell -- contains price: sell token ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token @@ -176,7 +326,6 @@ batch_protocol_fees AS ( SELECT solver, tx_hash, - -- sum(surplus * surplus_token_price) as surplus, sum(protocol_fee * protocol_fee_token_native_price) as protocol_fee, sum(network_fee_correction * network_fee_token_native_price) as network_fee_correction FROM diff --git a/src/sql/orderbook/prod_order_rewards.sql b/src/sql/orderbook/prod_order_rewards.sql index 229ea5ae..21bdd31e 100644 --- a/src/sql/orderbook/prod_order_rewards.sql +++ b/src/sql/orderbook/prod_order_rewards.sql @@ -79,20 +79,64 @@ order_surplus AS ( -- the bound for this table need to be a bit looser. AND ss.block_deadline <= {{end_block}} + 100 ), -order_protocol_fee AS ( +fee_policies_first_proxy as ( + select + auction_id, + order_uid, + max(application_order) as application_order, + count (*) as num_policies + from fee_policies + where auction_id in (select auction_id from order_surplus) + group by order_uid, auction_id +), +fee_policies_first as ( + select + fp.auction_id, + fp.order_uid, + fp.application_order, + fp.kind, + fp.surplus_factor, + fp.surplus_max_volume_factor, + fp.volume_factor, + fp.price_improvement_factor, + fp.price_improvement_max_volume_factor + from fee_policies_first_proxy fpmp join fee_policies fp on fp.auction_id = fpmp.auction_id and fp.order_uid = fpmp.order_uid and fp.application_order = fpmp.application_order +), +fee_policies_temp as ( + select + * + from fee_policies + where auction_id in (select auction_id from order_surplus) + except (select * from fee_policies_first ) +), +fee_policies_second as ( + select + * + from fee_policies_temp + UNION + select + auction_id, + order_uid, + 0 as application_order, + 'volume' as kind, + null as surplus_factor, + null as surplus_max_volume_factor, + 0 as volume_factor, + null as price_improvement_factor, + null as price_improvement_max_volume_factor + from fee_policies_first_proxy where num_policies = 1 +), +order_protocol_fee_first AS ( SELECT os.auction_id, - os.solver, - os.tx_hash, os.order_uid, os.sell_amount, os.buy_amount, - os.sell_token, - os.observed_fee, os.surplus, - os.surplus_token, + os.price_improvement, + os.kind, convert_from(os.app_data, 'UTF8')::JSONB->'metadata'->'partnerFee'->>'recipient' as partner_fee_recipient, - fp.kind as protocol_fee_kind, + fp.kind as protocol_fee_kind_first, CASE WHEN fp.kind = 'surplus' THEN CASE WHEN os.kind = 'sell' THEN @@ -139,11 +183,106 @@ order_protocol_fee AS ( WHEN os.kind = 'buy' THEN fp.volume_factor / (1 + fp.volume_factor) * os.sell_amount END + END AS protocol_fee_first, + os.surplus_token AS protocol_fee_token + FROM + order_surplus os + JOIN fee_policies_first fp -- contains protocol fee policy + ON os.auction_id = fp.auction_id + AND os.order_uid = fp.order_uid +), +order_surplus_intermediate as ( + select + auction_id, + order_uid, + CASE + WHEN kind = 'sell' then sell_amount + ELSE sell_amount - protocol_fee_first + END as sell_amount, + CASE + WHEN kind = 'sell' then buy_amount + protocol_fee_first + ELSE buy_amount + END as buy_amount, + surplus + protocol_fee_first as surplus, + price_improvement + protocol_fee_first as price_improvement, + protocol_fee_kind_first, + protocol_fee_first, + partner_fee_recipient + from order_protocol_fee_first +), +order_protocol_fee as ( + SELECT + os.auction_id, + os.solver, + os.tx_hash, + os.order_uid, + os.sell_amount, + os.buy_amount, + os.sell_token, + os.observed_fee, + os.surplus, + os.surplus_token, + protocol_fee_kind_first, + fp.kind as protocol_fee_kind_second, + protocol_fee_first, + CASE + WHEN fp.kind = 'surplus' THEN CASE + WHEN os.kind = 'sell' THEN + -- We assume that the case surplus_factor != 1 always. In + -- that case reconstructing the protocol fee would be + -- impossible anyways. This query will return a division by + -- zero error in that case. + protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 - fp.surplus_max_volume_factor) * osi.buy_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + fp.surplus_max_volume_factor / (1 + fp.surplus_max_volume_factor) * osi.sell_amount, + -- at most charge a fraction of volume + fp.surplus_factor / (1 - fp.surplus_factor) * osi.surplus -- charge a fraction of surplus + ) + END + WHEN fp.kind = 'priceimprovement' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 - fp.price_improvement_max_volume_factor) * osi.buy_amount, + -- charge a fraction of price improvement, at most 0 + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement + , + 0 + ) + ) + WHEN os.kind = 'buy' THEN protocol_fee_first + LEAST( + -- at most charge a fraction of volume + fp.price_improvement_max_volume_factor / (1 + fp.price_improvement_max_volume_factor) * osi.sell_amount, + -- charge a fraction of price improvement + GREATEST( + fp.price_improvement_factor / (1 - fp.price_improvement_factor) * osi.price_improvement, + 0 + ) + ) + END + WHEN fp.kind = 'volume' THEN CASE + WHEN os.kind = 'sell' THEN + protocol_fee_first + fp.volume_factor / (1 - fp.volume_factor) * osi.buy_amount + WHEN os.kind = 'buy' THEN + protocol_fee_first + fp.volume_factor / (1 + fp.volume_factor) * osi.sell_amount + END END AS protocol_fee, + osi.partner_fee_recipient, + CASE + WHEN osi.partner_fee_recipient IS NOT NULL THEN osi.protocol_fee_first + ELSE 0 + END AS partner_fee, os.surplus_token AS protocol_fee_token FROM order_surplus os - JOIN fee_policies fp -- contains protocol fee policy + JOIN order_surplus_intermediate osi + ON os.order_uid = osi.order_uid AND os.auction_id = osi.auction_id + JOIN fee_policies_second fp -- contains protocol fee policy ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid ), @@ -156,8 +295,12 @@ order_protocol_fee_prices AS ( opf.surplus, opf.protocol_fee, opf.protocol_fee_token, + CASE + WHEN opf.partner_fee_recipient IS NOT NULL THEN opf.protocol_fee_kind_second + ELSE opf.protocol_fee_kind_first + END AS protocol_fee_kind, + opf.partner_fee, opf.partner_fee_recipient, - opf.protocol_fee_kind, CASE WHEN opf.sell_token != opf.protocol_fee_token THEN (opf.sell_amount - opf.observed_fee) / opf.buy_amount * opf.protocol_fee ELSE opf.protocol_fee @@ -167,7 +310,7 @@ order_protocol_fee_prices AS ( ap_protocol.price / pow(10, 18) as protocol_fee_token_native_price, ap_sell.price / pow(10, 18) as network_fee_token_native_price FROM - order_protocol_fee opf + order_protocol_fee as opf JOIN auction_prices ap_sell -- contains price: sell token ON opf.auction_id = ap_sell.auction_id AND opf.sell_token = ap_sell.token @@ -222,7 +365,8 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - opfp.partner_fee_recipient as partner_fee_recipient, + opfp.partner_fee, + opfp.partner_fee_recipient, opfp.protocol_fee_kind from trade_hashes diff --git a/tests/unit/test_order_rewards_schema.py b/tests/unit/test_order_rewards_schema.py index 3de05930..9faa6477 100644 --- a/tests/unit/test_order_rewards_schema.py +++ b/tests/unit/test_order_rewards_schema.py @@ -27,6 +27,7 @@ def test_order_rewards_transformation(self): 12000000000000000, ], "quote_sell_token_price": [1.0, 250000000, 100000000000000.0], + "partner_fee": [0, 23123123123123, 0], "partner_fee_recipient": [None, "0x81", None], "protocol_fee_kind": [None, "volume", "priceimprovement"], } @@ -50,6 +51,7 @@ def test_order_rewards_transformation(self): "quote_buy_amount": "1000", "quote_gas_cost": 5000000000000000.15, "quote_sell_token_price": 1.0, + "partner_fee": "0", "partner_fee_recipient": None, "protocol_fee_kind": None, }, @@ -70,6 +72,7 @@ def test_order_rewards_transformation(self): "quote_buy_amount": "2000", "quote_gas_cost": 6000000000000000, "quote_sell_token_price": 250000000, + "partner_fee": "23123123123123", "partner_fee_recipient": "0x81", "protocol_fee_kind": "volume", }, @@ -90,6 +93,7 @@ def test_order_rewards_transformation(self): "quote_buy_amount": "10", "quote_gas_cost": 12000000000000000, "quote_sell_token_price": 100000000000000.0, + "partner_fee": "0", "partner_fee_recipient": None, "protocol_fee_kind": "priceimprovement", }, From f1587cc0413415fe4afb4d48547ba313ebd088fc Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Wed, 26 Jun 2024 14:50:39 +0200 Subject: [PATCH 2/2] fix test --- src/sql/orderbook/barn_order_rewards.sql | 2 +- src/sql/orderbook/prod_order_rewards.sql | 2 +- tests/integration/test_fetch_orderbook.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/sql/orderbook/barn_order_rewards.sql b/src/sql/orderbook/barn_order_rewards.sql index 21bdd31e..c18990af 100644 --- a/src/sql/orderbook/barn_order_rewards.sql +++ b/src/sql/orderbook/barn_order_rewards.sql @@ -365,7 +365,7 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - opfp.partner_fee, + cast(coalesce(opfp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, opfp.partner_fee_recipient, opfp.protocol_fee_kind from diff --git a/src/sql/orderbook/prod_order_rewards.sql b/src/sql/orderbook/prod_order_rewards.sql index 21bdd31e..c18990af 100644 --- a/src/sql/orderbook/prod_order_rewards.sql +++ b/src/sql/orderbook/prod_order_rewards.sql @@ -365,7 +365,7 @@ select cast(oq.buy_amount as numeric(78, 0)) :: text as quote_buy_amount, oq.gas_amount * oq.gas_price as quote_gas_cost, oq.sell_token_price as quote_sell_token_price, - opfp.partner_fee, + cast(coalesce(opfp.partner_fee, 0) as numeric(78, 0)) :: text as partner_fee, opfp.partner_fee_recipient, opfp.protocol_fee_kind from diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index 5e5d2026..3acf8ec7 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -146,6 +146,15 @@ def test_get_order_rewards(self): 1.0, 1.0, ], + "partner_fee": [ + "0", + "0", + "0", + "0", + "0", + "0", + "0", + ], "partner_fee_recipient": [ None, None, @@ -395,6 +404,11 @@ def test_get_order_rewards_with_integrator_fee(self): 0.000234609143374563, 0.000160802298220274, ], + "partner_fee": [ + "346011", + "0", + "0", + ], "partner_fee_recipient": [ "0x9FA3c00a92Ec5f96B1Ad2527ab41B3932EFEDa58", None,