Add protocol fee

queries/orderbook/batch_rewards.sql

@@ -28,13 +28,140 @@ WITH observed_settlements AS (SELECT
                                WHERE block_deadline >= {{start_block}}
                                  AND block_deadline <= {{end_block}}
                                GROUP BY ss.auction_id),
+-- protocol fees:
+order_surplus AS (
+    SELECT
+        ss.winner as solver,
+        at.auction_id,
+        s.tx_hash,
+        t.order_uid,
+        o.sell_token,
+        o.buy_token,
+        t.sell_amount, -- the total amount the user sends
+        t.buy_amount, -- the total amount the user receives
+        oe.surplus_fee as observed_fee, -- the total discrepancy between what the user sends and what they would have send if they traded at clearing price
+        o.kind,
+        CASE
+            WHEN o.kind = 'sell'
+                THEN t.buy_amount - t.sell_amount * o.buy_amount / (o.sell_amount + o.fee_amount)
+            WHEN o.kind = 'buy'
+                THEN t.buy_amount * (o.sell_amount + o.fee_amount) / o.buy_amount - t.sell_amount
+        END AS surplus,
+        CASE
+            WHEN o.kind = 'sell'
+                THEN o.buy_token
+            WHEN o.kind = 'buy'
+                THEN o.sell_token
+        END AS surplus_token
+    FROM settlements s -- links block_number and log_index to tx_from and tx_nonce
+    JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce
+        ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce
+    JOIN settlement_scores ss -- contains block_deadline
+        ON at.auction_id = ss.auction_id
+    JOIN trades t -- contains traded amounts
+        ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block
+    JOIN orders o -- contains tokens and limit amounts
+        ON t.order_uid = o.uid
+    JOIN order_execution oe -- contains surplus fee
+        ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id
+    WHERE ss.block_deadline >= {{start_block}}
+        AND ss.block_deadline <= {{end_block}}
+)
+,order_observations AS (
+    SELECT
+        os.auction_id,
+        os.solver,
+        os.tx_hash,
+        os.sell_amount,
+        os.buy_amount,
+        os.sell_token,
+        os.observed_fee,
+        os.surplus,
+        os.surplus_token,
+        CASE
+            WHEN fp.kind = 'surplus'
+                THEN
+                    CASE
+                        WHEN os.kind = 'sell'
+                            THEN
+                                CASE
+                                    WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero errors
+                                    -- We assume that the case surplus_factor != 1 always. In
+                                    -- that case reconstructing the protocol fee would be
+                                    -- impossible anyways. This query will return a division by
+                                    -- zero error in that case.
+                                        THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus
+                                    ELSE
+                                        LEAST(
+                                            fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume
+                                            fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
+                                        )
+                                END
+                        WHEN os.kind = 'buy'
+                            THEN
+                                CASE
+                                    WHEN fp.max_volume_factor = 1
+                                        THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus
+                                    ELSE
+                                        LEAST(
+                                            fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume
+                                            fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
+                                        )
+                                END
+                    END
+            WHEN fp.kind = 'volume'
+                THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount
+        END AS protocol_fee,
+        CASE
+            WHEN fp.kind = 'surplus'
+                THEN os.surplus_token
+            WHEN fp.kind = 'volume'
+                THEN os.sell_token
+        END AS protocol_fee_token
+    FROM order_surplus os
+    JOIN fee_policies fp -- contains protocol fee policy
+        ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid
+)
+,order_observations_prices AS (
+    SELECT
+        oo.solver,
+        oo.tx_hash,
+        oo.surplus,
+        oo.protocol_fee,
+        CASE
+            WHEN oo.sell_token != oo.protocol_fee_token
+                THEN (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee
+            ELSE oo.protocol_fee
+        END AS network_fee_correction,
+        oo.sell_token as network_fee_token,
+        ap_surplus.price / pow(10, 18) as surplus_token_price,
+        ap_protocol.price / pow(10, 18) as protocol_fee_token_price,
+        ap_sell.price / pow(10, 18) as network_fee_token_price
+    FROM order_observations oo
+    JOIN auction_prices ap_sell -- contains price: sell token
+        ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token
+    JOIN auction_prices ap_surplus -- contains price: surplus token
+        ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token
+    JOIN auction_prices ap_protocol -- contains price: protocol fee token
+        ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token
+),
+batch_protocol_fees AS (
+    SELECT
+        solver,
+        tx_hash,
+        -- sum(surplus * surplus_token_price) as surplus,
+        sum(protocol_fee * protocol_fee_token_price) as protocol_fee,
+        sum(network_fee_correction * network_fee_token_price) as network_fee_correction
+    FROM order_observations_prices oop
+    group by solver, tx_hash
+),
      reward_data AS (SELECT
                        -- observations
-                       tx_hash,
+                       os.tx_hash,
                        ss.auction_id,
                        -- TODO - Assuming that `solver == winner` when both not null
                        --  We will need to monitor that `solver == winner`!
-                       coalesce(solver, winner)               as solver,
+                       coalesce(os.solver, winner)               as solver,
                        block_number                           as settlement_block,
                        block_deadline,
                        case
@@ -50,26 +177,32 @@ WITH observed_settlements AS (SELECT
                        winning_score,
                        reference_score,
                        -- auction_participation
-                       participating_solvers
+                       participating_solvers,
+                       -- protocol_fees
+                       coalesce(cast(protocol_fee as numeric(78, 0)), 0) as protocol_fee,
+                       coalesce(cast(network_fee_correction as numeric(78, 0)), 0) as network_fee_correction
                      FROM settlement_scores ss
                             -- If there are reported scores,
                             -- there will always be a record of auction participants
                             JOIN auction_participation ap
                                  ON ss.auction_id = ap.auction_id
                        -- outer joins made in order to capture non-existent settlements.
                             LEFT OUTER JOIN observed_settlements os
-                                            ON os.auction_id = ss.auction_id),
+                                            ON os.auction_id = ss.auction_id
+                            LEFT OUTER JOIN batch_protocol_fees bpf
+                                ON bpf.tx_hash = os.tx_hash),
      reward_per_auction as (SELECT tx_hash,
                                    auction_id,
                                    settlement_block,
                                    block_deadline,
                                    solver,
                                    execution_cost,
                                    surplus,
-                                   fee,
-                                   surplus + fee - reference_score as uncapped_reward_eth,
+                                   protocol_fee + fee - network_fee_correction as fee,-- total fee for ranking
+                                   protocol_fee, -- the protocol fee
+                                   surplus + protocol_fee + fee - network_fee_correction - reference_score as uncapped_reward_eth,
                                    -- Uncapped Reward = CLAMP_[-E, E + exec_cost](uncapped_reward_eth)
-                                   LEAST(GREATEST(-{{EPSILON}}, surplus + fee - reference_score),
+                                   LEAST(GREATEST(-{{EPSILON}}, surplus + coalesce(protocol_fee - network_fee_correction, 0) + fee - reference_score),
                                      {{EPSILON}} + execution_cost) as capped_payment,
                                    winning_score,
                                    reference_score,
@@ -85,14 +218,21 @@ WITH observed_settlements AS (SELECT
                                 SUM(execution_cost) as exececution_cost_wei
                          FROM reward_per_auction rpt
                          GROUP BY solver),
+     protocol_fees as (SELECT solver,
+                                SUM(protocol_fee) as protocol_fee_wei
+                         FROM reward_per_auction rpt
+                         GROUP BY solver),
      aggregate_results as (SELECT concat('0x', encode(pc.solver, 'hex')) as solver,
                                   coalesce(payment_wei, 0)               as payment_eth,
                                   coalesce(exececution_cost_wei, 0)      as execution_cost_eth,
-                                  num_participating_batches
+                                  num_participating_batches,
+                                  coalesce(protocol_fee_wei, 0)      as protocol_fee_eth
                            FROM participation_counts pc
                                   LEFT OUTER JOIN primary_rewards pr
-                                                  ON pr.solver = pc.solver)
-
+                                                  ON pr.solver = pc.solver
+                                  LEFT OUTER JOIN protocol_fees pf
+                                                  ON pf.solver = pc.solver)
+--
 select *
 from aggregate_results
 order by solver;

queries/orderbook/protocol_fees.sql

@@ -0,0 +1,119 @@
+WITH order_surplus AS (
+    SELECT
+        ss.winner as solver,
+        at.auction_id,
+        t.order_uid,
+        o.sell_token, -- the total amount the user sends
+        o.buy_token,
+        t.sell_amount,
+        t.buy_amount,
+        oe.surplus_fee as observed_fee,
+        o.kind,
+        CASE
+            WHEN o.kind = 'sell'
+                THEN t.buy_amount - o.buy_amount / (o.sell_amount + o.fee_amount) * t.sell_amount
+            WHEN o.kind = 'buy'
+                THEN (o.sell_amount + o.fee_amount) / o.buy_amount * t.buy_amount - t.sell_amount
+        END AS surplus,
+        CASE
+            WHEN o.kind = 'sell'
+                THEN o.buy_token
+            WHEN o.kind = 'buy'
+                THEN o.sell_token
+        END AS surplus_token
+    FROM settlements s -- links block_number and log_index to tx_from and tx_nonce
+    JOIN auction_transaction at -- links auction_id to tx_from and tx_nonce
+        ON s.tx_from = at.tx_from AND s.tx_nonce = at.tx_nonce
+    JOIN settlement_scores ss -- contains block_deadline
+        ON at.auction_id = ss.auction_id
+    JOIN trades t -- contains traded amounts
+        ON s.block_number = t.block_number -- log_index cannot be checked, does not work correctly with multiple auctions on the same block
+    JOIN orders o -- contains tokens and limit amounts
+        ON t.order_uid = o.uid
+    JOIN order_execution oe -- contains surplus fee
+        ON t.order_uid = oe.order_uid AND at.auction_id = oe.auction_id
+    WHERE ss.block_deadline >= {{start_block}}
+        AND ss.block_deadline <= {{end_block}}
+)
+,order_observation AS (
+    SELECT
+        os.auction_id,
+        os.solver,
+        os.sell_amount, -- the total amount the user sends
+        os.buy_amount,
+        os.sell_token,
+        os.observed_fee,
+        os.surplus,
+        os.surplus_token,
+        CASE
+            WHEN fp.kind = 'surplus'
+                THEN
+                    CASE
+                        WHEN os.kind = 'sell'
+                            THEN
+                                CASE
+                                    WHEN fp.max_volume_factor = 1 -- this is done to avoid a division by zero bug
+                                        THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus
+                                    ELSE
+                                        LEAST(
+                                            fp.max_volume_factor / (1 - fp.max_volume_factor) * os.buy_amount, -- at most charge a fraction of volume
+                                            fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
+                                        )
+                                END
+                        WHEN os.kind = 'buy'
+                            THEN
+                                CASE
+                                    WHEN fp.max_volume_factor = 1
+                                        THEN fp.surplus_factor / (1 - fp.surplus_factor) * surplus
+                                    ELSE
+                                        LEAST(
+                                            fp.max_volume_factor / (1 - fp.max_volume_factor) * os.sell_amount, -- at most charge a fraction of volume
+                                            fp.surplus_factor / (1 - fp.surplus_factor) * surplus -- charge a fraction of surplus
+                                        )
+                                END
+                    END
+            WHEN fp.kind = 'volume'
+                THEN fp.volume_factor / (1 - fp.volume_factor) * os.sell_amount
+        END AS protocol_fee,
+        CASE
+            WHEN fp.kind = 'surplus'
+                THEN os.surplus_token
+            WHEN fp.kind = 'volume'
+                THEN os.sell_token
+        END AS protocol_fee_token
+    FROM order_surplus os
+    JOIN fee_policies fp -- contains protocol fee policy
+        ON os.auction_id = fp.auction_id AND os.order_uid = fp.order_uid
+)
+,order_observations_prices AS (
+    SELECT
+        oo.solver,
+        oo.surplus,
+        oo.protocol_fee,
+        CASE
+            WHEN oo.sell_token != oo.protocol_fee_token
+                THEN - (oo.sell_amount - oo.observed_fee) / oo.buy_amount * oo.protocol_fee
+            ELSE - oo.protocol_fee
+        END AS network_fee_correction,
+        oo.sell_token as network_fee_token,
+        ap_surplus.price / pow(10, 18) as surplus_token_price,
+        ap_protocol.price / pow(10, 18) as protocol_fee_token_price,
+        ap_sell.price / pow(10, 18) as network_fee_token_price
+    FROM order_observation oo
+    JOIN auction_prices ap_sell -- contains price: sell token
+        ON oo.auction_id = ap_sell.auction_id AND oo.sell_token = ap_sell.token
+    JOIN auction_prices ap_surplus -- contains price: surplus token
+        ON oo.auction_id = ap_surplus.auction_id AND oo.surplus_token = ap_surplus.token
+    JOIN auction_prices ap_protocol -- contains price: protocol fee token
+        ON oo.auction_id = ap_protocol.auction_id AND oo.protocol_fee_token = ap_protocol.token
+),
+batch_aggregate AS (
+    SELECT
+        concat('0x', encode(solver, 'hex')) as solver,
+        -- sum(surplus * surplus_token_price) as surplus,
+        sum(protocol_fee * protocol_fee_token_price) as protocol_fee,
+        sum(network_fee_correction * network_fee_token_price) as network_fee_correction
+    FROM order_observations_prices oop
+    group by solver
+)
+SELECT * FROM batch_aggregate

src/fetch/payouts.py

@@ -33,6 +33,7 @@
     "reward_cow",
     "secondary_reward_cow",
     "quote_reward_cow",
+    "protocol_fee_eth",
 }
 SLIPPAGE_COLUMNS = {
     "solver",
@@ -79,10 +80,13 @@ def __init__(  # pylint: disable=too-many-arguments
         primary_reward_cow: int,
         secondary_reward_cow: int,
         quote_reward_cow: int,
+        protocol_fee_eth: int,
     ):
         assert exec_cost >= 0, f"invalid execution cost {exec_cost}"
         assert secondary_reward_eth >= 0, "invalid secondary_reward_eth"
         assert secondary_reward_cow >= 0, "invalid secondary_reward_cow"
+        assert quote_reward_cow >= 0, "invalid quote reward"
+        assert protocol_fee_eth >= 0, "invalid protocol_fee_eth"
 
         self.solver = solver
         self.solver_name = solver_name
@@ -94,6 +98,7 @@ def __init__(  # pylint: disable=too-many-arguments
         self.secondary_reward_eth = secondary_reward_eth
         self.secondary_reward_cow = secondary_reward_cow
         self.quote_reward_cow = quote_reward_cow
+        self.protocol_fee_eth = protocol_fee_eth
 
     @classmethod
     def from_series(cls, frame: Series) -> RewardAndPenaltyDatum:
@@ -120,11 +125,17 @@ def from_series(cls, frame: Series) -> RewardAndPenaltyDatum:
             secondary_reward_eth=int(frame["secondary_reward_eth"]),
             secondary_reward_cow=int(frame["secondary_reward_cow"]),
             quote_reward_cow=int(frame["quote_reward_cow"]),
+            protocol_fee_eth=int(frame["protocol_fee_eth"]),
         )
 
     def total_outgoing_eth(self) -> int:
         """Total outgoing amount (including slippage) for the payout."""
-        return self.payment_eth + self.secondary_reward_eth + self.slippage_eth
+        return (
+            self.payment_eth
+            + self.secondary_reward_eth
+            + self.slippage_eth
+            - self.protocol_fee_eth
+        )
 
     def total_cow_reward(self) -> int:
         """Total outgoing COW token reward"""
@@ -161,7 +172,9 @@ def as_payouts(self) -> list[Transfer]:
         total_eth_reward = int(self.total_eth_reward())
         total_cow_reward = int(self.total_cow_reward())
 
-        reimbursement_eth = int(self.exec_cost + self.slippage_eth)
+        reimbursement_eth = int(
+            self.exec_cost + self.slippage_eth - self.protocol_fee_eth
+        )
         # We do not have access to token conversion here, but we do have other converted values
         # x_eth:x_cow = y_eth:y_cow --> y_cow = y_eth * x_cow / x_eth
         reimbursement_cow = (