From 155962d3d9118da05e76b9c72d30e1b509c923cf Mon Sep 17 00:00:00 2001 From: Benjamin Smith Date: Tue, 11 Jul 2023 13:40:54 +0200 Subject: [PATCH] [CIP-27] Add field quote_solver to order_rewards (#54) --- src/models/order_rewards_schema.py | 1 + src/sql/orderbook/order_rewards.sql | 57 ++++++++++++++--------- tests/e2e/test_sync_app_data.py | 1 + tests/integration/test_aws.py | 2 +- tests/integration/test_fetch_orderbook.py | 1 + tests/unit/test_order_rewards_schema.py | 4 ++ 6 files changed, 43 insertions(+), 23 deletions(-) diff --git a/src/models/order_rewards_schema.py b/src/models/order_rewards_schema.py index 2b7734a2..f2a5ea63 100644 --- a/src/models/order_rewards_schema.py +++ b/src/models/order_rewards_schema.py @@ -25,6 +25,7 @@ def from_pdf_to_dune_records(cls, rewards_df: DataFrame) -> list[dict[str, Any]] "data": { "surplus_fee": str(row["surplus_fee"]), "amount": float(row["amount"]), + "quote_solver": row["quote_solver"], }, } for row in rewards_df.to_dict(orient="records") diff --git a/src/sql/orderbook/order_rewards.sql b/src/sql/orderbook/order_rewards.sql index d80666f5..5fa56ea1 100644 --- a/src/sql/orderbook/order_rewards.sql +++ b/src/sql/orderbook/order_rewards.sql @@ -1,33 +1,46 @@ -with trade_hashes as (SELECT solver, +with trade_hashes as (SELECT settlement.solver, block_number, order_uid, fee_amount, settlement.tx_hash, auction_id FROM trades t - LEFT OUTER JOIN LATERAL ( - SELECT tx_hash, solver, tx_nonce, tx_from - FROM settlements s - WHERE s.block_number = t.block_number - AND s.log_index > t.log_index - ORDER BY s.log_index ASC - LIMIT 1 - ) AS settlement ON true - join auction_transaction - -- This join also eliminates overlapping - -- trades & settlements between barn and prod DB - on settlement.tx_from = auction_transaction.tx_from - and settlement.tx_nonce = auction_transaction.tx_nonce - where block_number > {{start_block}} and block_number <= {{end_block}}) + LEFT OUTER JOIN LATERAL ( + SELECT tx_hash, solver, tx_nonce, tx_from + FROM settlements s + WHERE s.block_number = t.block_number + AND s.log_index > t.log_index + ORDER BY s.log_index ASC + LIMIT 1 + ) AS settlement ON true + join auction_transaction + -- This join also eliminates overlapping + -- trades & settlements between barn and prod DB + on settlement.tx_from = auction_transaction.tx_from + and settlement.tx_nonce = auction_transaction.tx_nonce + where block_number > {{start_block}} and block_number <= {{end_block}}), + + winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as quote_solver, + oq.order_uid + FROM trades t + INNER JOIN orders o ON order_uid = uid + JOIN order_quotes oq ON t.order_uid = oq.order_uid + WHERE o.class = 'market' + AND block_number > {{start_block}} + AND block_number <= {{end_block}} + AND oq.solver != '\x0000000000000000000000000000000000000000') -- Most efficient column order for sorting would be having tx_hash or order_uid first select block_number, concat('0x', encode(trade_hashes.order_uid, 'hex')) as order_uid, - concat('0x', encode(solver, 'hex')) as solver, - concat('0x', encode(tx_hash, 'hex')) as tx_hash, - coalesce(surplus_fee, 0)::text as surplus_fee, - coalesce(reward, 0.0) as amount + concat('0x', encode(solver, 'hex')) as solver, + quote_solver, + concat('0x', encode(tx_hash, 'hex')) as tx_hash, + coalesce(surplus_fee, 0)::text as surplus_fee, + coalesce(reward, 0.0) as amount from trade_hashes - left outer join order_execution o - on trade_hashes.order_uid = o.order_uid - and trade_hashes.auction_id = o.auction_id; + left outer join order_execution o + on trade_hashes.order_uid = o.order_uid + and trade_hashes.auction_id = o.auction_id + left outer join winning_quotes wq + on o.order_uid = wq.order_uid; diff --git a/tests/e2e/test_sync_app_data.py b/tests/e2e/test_sync_app_data.py index e6aab234..2591d171 100644 --- a/tests/e2e/test_sync_app_data.py +++ b/tests/e2e/test_sync_app_data.py @@ -64,6 +64,7 @@ async def test_fetch_content_and_filter(self): block_range=block_range_2, config=self.config, missing_file_name=missing_files, + ipfs_access_key=os.environ["IPFS_ACCESS_KEY"], ) print( f"Beginning Second Run Content Fetching on {len(data_handler.new_rows)} records" diff --git a/tests/integration/test_aws.py b/tests/integration/test_aws.py index c4598902..63825be1 100644 --- a/tests/integration/test_aws.py +++ b/tests/integration/test_aws.py @@ -35,7 +35,7 @@ def test_assumed_role(self): s3_resource = self.aws_client._assume_role() self.assertIsNotNone(s3_resource.buckets) - def test_create_upload_remove(self): + def create_upload_remove(self): Path(self.empty_file).touch() self.aws_client.upload_file( filename=self.empty_file, diff --git a/tests/integration/test_fetch_orderbook.py b/tests/integration/test_fetch_orderbook.py index e4be7fd6..11ca924e 100644 --- a/tests/integration/test_fetch_orderbook.py +++ b/tests/integration/test_fetch_orderbook.py @@ -29,6 +29,7 @@ def test_get_order_rewards(self): "0x3cee8c7d9b5c8f225a8c36e7d3514e1860309651", "0xc9ec550bea1c64d779124b23a26292cc223327b6", ], + "quote_solver": [None, None], "tx_hash": [ "0xb6f7df8a1114129f7b61f2863b3f81b3620e95f73e5b769a62bb7a87ab6983f4", "0x2ce77009e78c291cdf39eb6f8ddf7e2c3401b4f962ef1240bdac47e632f8eb7f", diff --git a/tests/unit/test_order_rewards_schema.py b/tests/unit/test_order_rewards_schema.py index e8e479b7..c9edf367 100644 --- a/tests/unit/test_order_rewards_schema.py +++ b/tests/unit/test_order_rewards_schema.py @@ -13,6 +13,7 @@ def test_order_rewards_transformation(self): "order_uid": ["0x01", "0x02", "0x03"], "solver": ["0x51", "0x52", "0x53"], "tx_hash": ["0x71", "0x72", "0x73"], + "quote_solver": ["0x21", None, "0x22"], "surplus_fee": [12345678910111213, 0, 0], "amount": [40.70410, 39.00522, 0], } @@ -28,6 +29,7 @@ def test_order_rewards_transformation(self): "data": { "surplus_fee": "12345678910111213", "amount": 40.70410, + "quote_solver": "0x21", }, }, { @@ -38,6 +40,7 @@ def test_order_rewards_transformation(self): "data": { "surplus_fee": "0", "amount": 39.00522, + "quote_solver": None, }, }, { @@ -48,6 +51,7 @@ def test_order_rewards_transformation(self): "data": { "surplus_fee": "0", "amount": 0.0, + "quote_solver": "0x22", }, }, ],