From 648633cb799a5723fb852ac3a10e1e75f0c79d1a Mon Sep 17 00:00:00 2001 From: hweawer Date: Mon, 12 Aug 2024 15:57:12 +0200 Subject: [PATCH] Release mellow main (#254) * chore: set default strategy * Upgrade python to 3.12 * Upgrade poetry * upgrade curl * Mellow mvp (#224) * Mellow direct deposit, skip ABI * Prepare direct deposit transaction * Property checking * Fix linter * amount parameter in the contract * Fix comments * Check variable not set * Contract abi integration test * Add fixtures * Remove amount parameter from abi * Formatter changes * Formatter changes * Holesky mark in tests * Test balanceOf of weth * Renamings * Rename mark * block_identifier * Update src/variables.py Co-authored-by: Raman Siamionau * Fix comments * Renamed env var in pipeline * Change module to deposit * Send mellow transaction * Change comment * DD description * Update README.md Co-authored-by: Raman Siamionau * Refactor sending mellow transaction * Refactor sending mellow transaction * is_mellow_depositable unit test * Unit test for sending mellow tx * Formatting --------- Co-authored-by: Raman Siamionau * Metric for modules (#229) * Expose modules metric * Reorder * Reorder * Update src/metrics/metrics.py Co-authored-by: Raman Siamionau --------- Co-authored-by: Raman Siamionau * Log env vars (#227) * Log public env vars * Add chain_id * Public env vars in prometheus metric * Add message to a log * Change info description * Update src/variables.py Co-authored-by: Raman Siamionau * Update src/metrics/metrics.py Co-authored-by: Raman Siamionau * Remove prefix, not assert * Fix imports --------- Co-authored-by: Raman Siamionau * Fix Info metircs endpoint(#231) * Fix log string * Change forematting * Convert values to strings * remove convertion * Mellow deposit strategy (#230) * Change validation for mellow deposits * Load WQ contract from the locator * Refactor * Change log message * Beffered ether * Change ABI * Move check inside is_mellow_depositable * Add ping type to the rabbit messages in the unvetter * Mellow new ABIs * Remove old build metrics (#239) * Add mellow variable to examples (#237) * Add mellow variable to examples * Remove import * Update holesky address * Add account to variables (#241) * Add account to metrics * Fix field ref * Change to propery * Direct access property * Separate mellow flow (#235) * Separate mellow flow * Formatting * Rerun integration tests * Strategy return * Rewrite to abstract classes * Fix formatting * Fix bug with return * Restructure * remove init * _is_mellow * Fix tests * Fix signs test * Fix integration * Fix unit test * Mellow test * Update src/blockchain/deposit_strategy/base_deposit_strategy.py Co-authored-by: Raman Siamionau * Refactor * Renamings * Imports * inject dependecies --------- Co-authored-by: Raman Siamionau * Metric per module (#244) * Per module metric * FOrmatting * Fix error repr * Metrics for checks (#246) * Metrics for different check statuses * Formatting * Fix early returns (#248) * Mellow fallback (#250) * Mellow fallback * Fix unit tests * Fix DepositorBot constructor in test * Fix positional argument * Double max deposit count * Fix test * Improve test * Sender chain in integrations * Without sender chain * Reorder * Remove redundant function * New metric for mellow * Cast * Merge main --------- Co-authored-by: F4ever --- .../contracts/deposit_security_module.py | 2 + .../deposit_strategy/base_deposit_strategy.py | 16 ++++- .../deposit_transaction_sender.py | 2 +- .../deposit_strategy/gas_price_calculator.py | 35 ++++++----- src/bots/depositor.py | 58 +++++++++++++------ src/metrics/metrics.py | 39 ++++++++++++- .../test_base_deposit_strategy.py | 7 ++- .../test_deposit_transaction_sender.py | 3 +- tests/bots/test_depositor.py | 29 ++++++++-- tests/fixtures/__init__.py | 2 +- tests/fixtures/strategy.py | 4 +- 11 files changed, 149 insertions(+), 48 deletions(-) diff --git a/src/blockchain/contracts/deposit_security_module.py b/src/blockchain/contracts/deposit_security_module.py index 10244720..d141a51d 100644 --- a/src/blockchain/contracts/deposit_security_module.py +++ b/src/blockchain/contracts/deposit_security_module.py @@ -2,6 +2,7 @@ from blockchain.contracts.base_interface import ContractInterface from eth_typing import ChecksumAddress, Hash32 +from metrics.metrics import CAN_DEPOSIT from web3.contract.contract import ContractFunction from web3.exceptions import ABIFunctionNotFound, ContractLogicError from web3.types import BlockIdentifier @@ -37,6 +38,7 @@ def can_deposit(self, staking_module_id: int, block_identifier: BlockIdentifier """ response = self.functions.canDeposit(staking_module_id).call(block_identifier=block_identifier) logger.info({'msg': f'Call `canDeposit({staking_module_id})`.', 'value': response, 'block_identifier': repr(block_identifier)}) + CAN_DEPOSIT.labels(staking_module_id).set(int(response)) return response def deposit_buffered_ether( diff --git a/src/blockchain/deposit_strategy/base_deposit_strategy.py b/src/blockchain/deposit_strategy/base_deposit_strategy.py index 4a635df7..780283a2 100644 --- a/src/blockchain/deposit_strategy/base_deposit_strategy.py +++ b/src/blockchain/deposit_strategy/base_deposit_strategy.py @@ -28,7 +28,7 @@ def deposited_keys_amount(self, module_id: int) -> int: module_id, depositable_ether, ) - POSSIBLE_DEPOSITS_AMOUNT.labels(module_id).set(possible_deposits_amount) + POSSIBLE_DEPOSITS_AMOUNT.labels(module_id, 0).set(possible_deposits_amount) return possible_deposits_amount @@ -44,3 +44,17 @@ def _depositable_ether(self) -> Wei: logger.info({'msg': 'Adding mellow vault balance to the depositable check', 'vault': additional_ether}) depositable_ether += additional_ether return depositable_ether + + def deposited_keys_amount(self, module_id: int) -> int: + depositable_ether = self._depositable_ether() + possible_deposits_amount_assumption = self.w3.lido.staking_router.get_staking_module_max_deposits_count( + module_id, + depositable_ether, + ) + possible_deposited_eth = Web3.to_wei(32 * possible_deposits_amount_assumption, 'ether') + possible_deposits_amount = self.w3.lido.staking_router.get_staking_module_max_deposits_count( + module_id, + possible_deposited_eth, + ) + POSSIBLE_DEPOSITS_AMOUNT.labels(module_id, 1).set(possible_deposits_amount) + return possible_deposits_amount if possible_deposits_amount_assumption == possible_deposits_amount else 0 diff --git a/src/blockchain/deposit_strategy/deposit_transaction_sender.py b/src/blockchain/deposit_strategy/deposit_transaction_sender.py index 1949848c..cc017c02 100644 --- a/src/blockchain/deposit_strategy/deposit_transaction_sender.py +++ b/src/blockchain/deposit_strategy/deposit_transaction_sender.py @@ -29,8 +29,8 @@ def _prepare_signs_for_deposit(quorum: list[DepositMessage]) -> tuple[tuple[str, def prepare_and_send( self, quorum: list[DepositMessage], - with_flashbots: bool, is_mellow: bool, + with_flashbots: bool, ) -> bool: tx = self._prepare_mellow_tx(quorum) if is_mellow else self._prepare_general_tx(quorum) return self._send_transaction(tx, with_flashbots) diff --git a/src/blockchain/deposit_strategy/gas_price_calculator.py b/src/blockchain/deposit_strategy/gas_price_calculator.py index 1afa83dd..1186cf2e 100644 --- a/src/blockchain/deposit_strategy/gas_price_calculator.py +++ b/src/blockchain/deposit_strategy/gas_price_calculator.py @@ -8,7 +8,7 @@ from blockchain.deposit_strategy.base_deposit_strategy import BaseDepositStrategy from blockchain.typings import Web3 from eth_typing import BlockNumber -from metrics.metrics import GAS_FEE +from metrics.metrics import DEPOSIT_AMOUNT_OK, GAS_FEE, GAS_OK from web3.types import Wei logger = logging.getLogger(__name__) @@ -30,12 +30,14 @@ def is_gas_price_ok(self, module_id: int) -> bool: current_buffered_ether = self.w3.lido.lido.get_depositable_ether() if current_buffered_ether > variables.MAX_BUFFERED_ETHERS: - return current_gas_fee <= variables.MAX_GAS_FEE - - recommended_gas_fee = self._get_recommended_gas_fee() - GAS_FEE.labels('recommended_fee', module_id).set(recommended_gas_fee) - GAS_FEE.labels('max_fee', module_id).set(variables.MAX_GAS_FEE) - return recommended_gas_fee >= current_gas_fee + success = current_gas_fee <= variables.MAX_GAS_FEE + else: + recommended_gas_fee = self._get_recommended_gas_fee() + GAS_FEE.labels('recommended_fee', module_id).set(recommended_gas_fee) + GAS_FEE.labels('max_fee', module_id).set(variables.MAX_GAS_FEE) + success = recommended_gas_fee >= current_gas_fee + GAS_OK.labels(module_id).set(int(success)) + return success def _get_pending_base_fee(self) -> Wei: base_fee_per_gas = self.w3.eth.get_block('pending')['baseFeePerGas'] @@ -44,6 +46,7 @@ def _get_pending_base_fee(self) -> Wei: def calculate_deposit_recommendation(self, deposit_strategy: BaseDepositStrategy, module_id: int) -> bool: possible_keys = deposit_strategy.deposited_keys_amount(module_id) + success = False if possible_keys < deposit_strategy.DEPOSITABLE_KEYS_THRESHOLD: logger.info( { @@ -52,15 +55,15 @@ def calculate_deposit_recommendation(self, deposit_strategy: BaseDepositStrategy 'threshold': deposit_strategy.DEPOSITABLE_KEYS_THRESHOLD, } ) - return False - - recommended_max_gas = GasPriceCalculator._calculate_recommended_gas_based_on_deposit_amount( - possible_keys, - module_id, - ) - base_fee_per_gas = self._get_pending_base_fee() - success = recommended_max_gas >= base_fee_per_gas - logger.info({'msg': 'Calculations deposit recommendations.', 'value': success}) + else: + recommended_max_gas = GasPriceCalculator._calculate_recommended_gas_based_on_deposit_amount( + possible_keys, + module_id, + ) + base_fee_per_gas = self._get_pending_base_fee() + success = recommended_max_gas >= base_fee_per_gas + logger.info({'msg': 'Calculations deposit recommendations.', 'value': success}) + DEPOSIT_AMOUNT_OK.labels(module_id).set(int(success)) return success @staticmethod diff --git a/src/bots/depositor.py b/src/bots/depositor.py index a221bc02..23744453 100644 --- a/src/bots/depositor.py +++ b/src/bots/depositor.py @@ -14,8 +14,10 @@ from metrics.metrics import ( ACCOUNT_BALANCE, CURRENT_QUORUM_SIZE, + IS_DEPOSITABLE, MELLOW_VAULT_BALANCE, MODULE_TX_SEND, + QUORUM, UNEXPECTED_EXCEPTIONS, ) from metrics.transport_message_metrics import message_metrics_filter @@ -33,8 +35,8 @@ def run_depositor(w3): logger.info({'msg': 'Initialize Depositor bot.'}) - gas_price_calculator = GasPriceCalculator(w3) sender = Sender(w3) + gas_price_calculator = GasPriceCalculator(w3) mellow_deposit_strategy = MellowDepositStrategy(w3) base_deposit_strategy = BaseDepositStrategy(w3) depositor_bot = DepositorBot(w3, sender, gas_price_calculator, mellow_deposit_strategy, base_deposit_strategy) @@ -66,8 +68,8 @@ def __init__( base_deposit_strategy: BaseDepositStrategy, ): self.w3 = w3 - self._gas_price_calculator = gas_price_calcaulator self._sender = sender + self._gas_price_calculator = gas_price_calcaulator self._mellow_strategy = mellow_deposit_strategy self._general_strategy = base_deposit_strategy @@ -137,7 +139,7 @@ def _is_mellow_depositable( { 'msg': 'Mellow module check failed.', 'contract_module': staking_module_contract.get_staking_module_id(), - 'tx_module': module_id + 'tx_module': module_id, } ) return False @@ -145,15 +147,21 @@ def _is_mellow_depositable( except Exception as e: logger.warning( { - 'msg': 'Failed to check if mellow depositable', + 'msg': 'Failed to check if mellow depositable.', 'module_id': module_id, - 'err': repr(e) + 'err': repr(e), } ) return False MELLOW_VAULT_BALANCE.labels(module_id).set(balance) if balance < variables.VAULT_DIRECT_DEPOSIT_THRESHOLD: - logger.info({'msg': f'{balance} is less than VAULT_DIRECT_DEPOSIT_THRESHOLD while building mellow transaction.'}) + logger.info( + { + 'msg': f'{balance} is less than VAULT_DIRECT_DEPOSIT_THRESHOLD while building mellow transaction.', + 'balance': balance, + 'threshold': variables.VAULT_DIRECT_DEPOSIT_THRESHOLD, + } + ) return False logger.debug({'msg': 'Mellow module check succeeded.', 'tx_module': module_id}) return True @@ -186,22 +194,25 @@ def _deposit_to_module(self, module_id: int) -> bool: if is_depositable and quorum and can_deposit and gas_is_ok and is_deposit_amount_ok: logger.info({'msg': 'Checks passed. Prepare deposit tx.', 'is_mellow': is_mellow}) - success = self.prepare_and_send_tx(quorum, is_mellow, self._flashbots_works) + success = self.prepare_and_send_tx(module_id, quorum, is_mellow) + if not success and is_mellow: + success = self.prepare_and_send_tx(module_id, quorum, False) self._flashbots_works = not self._flashbots_works or success - self._mellow_works = success return success logger.info({'msg': 'Checks failed. Skip deposit.'}) return False def _select_strategy(self, module_id) -> tuple[BaseDepositStrategy, bool]: - if self._mellow_works and self._is_mellow_depositable(module_id): + if self._is_mellow_depositable(module_id): return self._mellow_strategy, True return self._general_strategy, False def _check_module_status(self, module_id: int) -> bool: """Returns True if module is ready for deposit""" - return self.w3.lido.staking_router.is_staking_module_active(module_id) + ready = self.w3.lido.staking_router.is_staking_module_active(module_id) + IS_DEPOSITABLE.labels(module_id).set(int(ready)) + return ready def _get_quorum(self, module_id: int) -> Optional[list[DepositMessage]]: """Returns quorum messages or None is quorum is not ready""" @@ -230,11 +241,13 @@ def _get_quorum(self, module_id: int) -> Optional[list[DepositMessage]]: if quorum_size >= min_signs_to_deposit: CURRENT_QUORUM_SIZE.labels('current').set(quorum_size) + QUORUM.labels(module_id).set(1) return list(unified_messages) max_quorum_size = max(quorum_size, max_quorum_size) CURRENT_QUORUM_SIZE.labels('current').set(max_quorum_size) + QUORUM.labels(module_id).set(0) def _get_message_actualize_filter(self) -> Callable[[DepositMessage], bool]: latest = self.w3.eth.get_block('latest') @@ -275,13 +288,24 @@ def message_filter(message: DepositMessage) -> bool: return message_filter - def prepare_and_send_tx(self, quorum: list[DepositMessage], is_mellow: bool, module_id: int) -> bool: - success = self._sender.prepare_and_send( - quorum, - self._flashbots_works, - is_mellow, - ) + def prepare_and_send_tx(self, module_id: int, quorum: list[DepositMessage], is_mellow: bool) -> bool: + if is_mellow: + try: + success = self._sender.prepare_and_send( + quorum, + is_mellow, + self._flashbots_works, + ) + except Exception as e: + success = False + logger.warning({'msg': 'Error while sending mellow transaction', 'err': repr(e)}) + else: + success = self._sender.prepare_and_send( + quorum, + is_mellow, + self._flashbots_works, + ) logger.info({'msg': f'Tx send. Result is {success}.'}) label = 'success' if success else 'failure' - MODULE_TX_SEND.labels(label, module_id).inc() + MODULE_TX_SEND.labels(label, module_id, int(is_mellow)).inc() return success diff --git a/src/metrics/metrics.py b/src/metrics/metrics.py index ea53f88e..07a79a7d 100644 --- a/src/metrics/metrics.py +++ b/src/metrics/metrics.py @@ -12,7 +12,7 @@ MODULE_TX_SEND = Counter( 'transactions', 'Amount of send transaction from bot with per module distribution.', - ['status', 'module_id'], + ['status', 'module_id', 'is_mellow'], namespace=PROMETHEUS_PREFIX ) @@ -55,7 +55,7 @@ POSSIBLE_DEPOSITS_AMOUNT = Gauge( 'possible_deposits_amount', 'Possible deposits amount.', - ['module_id'], + ['module_id', 'is_mellow'], namespace=PROMETHEUS_PREFIX, ) @@ -66,6 +66,41 @@ namespace=PROMETHEUS_PREFIX, ) +IS_DEPOSITABLE = Gauge( + 'is_depositable', + 'Represents is_depositable check.', + ['module_id'], + namespace=PROMETHEUS_PREFIX, +) + +QUORUM = Gauge( + 'quorum', + 'Represents if quorum could be collected.', + ['module_id'], + namespace=PROMETHEUS_PREFIX, +) + +CAN_DEPOSIT = Gauge( + 'can_deposit', + 'Represents can_deposit check.', + ['module_id'], + namespace=PROMETHEUS_PREFIX, +) + +GAS_OK = Gauge( + 'is_gas_ok', + 'Represents is_gas_ok check.', + ['module_id'], + namespace=PROMETHEUS_PREFIX, +) + +DEPOSIT_AMOUNT_OK = Gauge( + 'is_deposit_amount_ok', + 'Represents is_deposit_amount_ok check.', + ['module_id'], + namespace=PROMETHEUS_PREFIX, +) + ETH_RPC_REQUESTS_DURATION = Histogram('eth_rpc_requests_duration', 'Duration of requests to ETH1 RPC', namespace=PROMETHEUS_PREFIX) ETH_RPC_REQUESTS = Counter( diff --git a/tests/blockchain/deposit_strategy/test_base_deposit_strategy.py b/tests/blockchain/deposit_strategy/test_base_deposit_strategy.py index 8b01cfb3..44b60cba 100644 --- a/tests/blockchain/deposit_strategy/test_base_deposit_strategy.py +++ b/tests/blockchain/deposit_strategy/test_base_deposit_strategy.py @@ -1,6 +1,7 @@ from unittest.mock import Mock import pytest +from web3 import Web3 MODULE_ID = 1 @@ -32,7 +33,11 @@ def test_deposited_keys_amount_mellow(mellow_deposit_strategy): mellow_deposit_strategy.w3.lido.staking_router.get_staking_module_max_deposits_count = Mock(return_value=possible_deposits) assert mellow_deposit_strategy.deposited_keys_amount(MODULE_ID) == possible_deposits - mellow_deposit_strategy.w3.lido.staking_router.get_staking_module_max_deposits_count.assert_called_once_with( + mellow_deposit_strategy.w3.lido.staking_router.get_staking_module_max_deposits_count.assert_any_call( MODULE_ID, depositable_eth + vault_balance, ) + mellow_deposit_strategy.w3.lido.staking_router.get_staking_module_max_deposits_count.assert_any_call( + MODULE_ID, + Web3.to_wei(32 * possible_deposits, 'ether'), + ) diff --git a/tests/blockchain/deposit_strategy/test_deposit_transaction_sender.py b/tests/blockchain/deposit_strategy/test_deposit_transaction_sender.py index eb201c86..dcecdaaa 100644 --- a/tests/blockchain/deposit_strategy/test_deposit_transaction_sender.py +++ b/tests/blockchain/deposit_strategy/test_deposit_transaction_sender.py @@ -1,13 +1,14 @@ from unittest.mock import Mock import pytest +from blockchain.deposit_strategy.deposit_transaction_sender import Sender from transport.msg_types.deposit import DepositMessage MODULE_ID = 1 @pytest.mark.unit -def test_send_deposit_tx_not_mellow(deposit_transaction_sender): +def test_send_deposit_tx_not_mellow(deposit_transaction_sender: Sender): deposit_transaction_sender._w3.transaction.check = Mock(return_value=False) messages = [DepositMessage( type='deposit', diff --git a/tests/bots/test_depositor.py b/tests/bots/test_depositor.py index 239e47e6..92cdce90 100644 --- a/tests/bots/test_depositor.py +++ b/tests/bots/test_depositor.py @@ -298,7 +298,7 @@ def add_accounts_to_guardian(web3_lido_integration, set_integration_account): [[19628126, 1], [19628126, 2]], indirect=['web3_provider_integration'], ) -def test_depositor_bot( +def test_depositor_bot_non_mellow_deposits( web3_provider_integration, web3_lido_integration, deposit_transaction_sender_integration, @@ -308,7 +308,12 @@ def test_depositor_bot( module_id, add_accounts_to_guardian, ): + # Disable mellow integration + variables.MELLOW_CONTRACT_ADDRESS = None + # Define the whitelist of deposit modules variables.DEPOSIT_MODULES_WHITELIST = [1, 2] + + # Set the balance for the first account web3_lido_integration.provider.make_request( 'anvil_setBalance', [ @@ -317,6 +322,7 @@ def test_depositor_bot( ], ) + # Submit multiple transactions for _ in range(15): web3_lido_integration.lido.lido.functions.submit(web3_lido_integration.eth.accounts[0]).transact( { @@ -325,18 +331,26 @@ def test_depositor_bot( } ) + # Set the maximum number of deposits web3_lido_integration.lido.deposit_security_module.functions.setMaxDeposits(100).transact({'from': DSM_OWNER}) + # Get the latest block latest = web3_lido_integration.eth.get_block('latest') + # Get the current nonce for the staking module old_module_nonce = web3_lido_integration.lido.staking_router.get_staking_module_nonce(module_id) - deposit_message_1 = get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_1, COUNCIL_PK_1, module_id) - deposit_message_2 = get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_1, COUNCIL_PK_1, module_id) - deposit_message_3 = get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_2, COUNCIL_PK_2, module_id) + # Create deposit messages + deposit_messages = [ + get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_1, COUNCIL_PK_1, module_id), + get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_1, COUNCIL_PK_1, module_id), + get_deposit_message(web3_lido_integration, COUNCIL_ADDRESS_2, COUNCIL_PK_2, module_id), + ] + # Mine a new block web3_lido_integration.provider.make_request('anvil_mine', [1]) + # Initialize the DepositorBot db: DepositorBot = DepositorBot( web3_lido_integration, deposit_transaction_sender_integration, @@ -344,12 +358,15 @@ def test_depositor_bot( mellow_deposit_strategy_integration, base_deposit_strategy_integration, ) - db._mellow_works = False + + # Clear the message storage and execute the bot without any messages db.message_storage.messages = [] db.execute(latest) + # Assert that the staking module nonce has not changed assert web3_lido_integration.lido.staking_router.get_staking_module_nonce(module_id) == old_module_nonce - db.message_storage.messages = [deposit_message_1, deposit_message_2, deposit_message_3] + # Execute the bot with deposit messages and assert that the nonce has increased by 1 + db.message_storage.messages = deposit_messages assert db.execute(latest) assert web3_lido_integration.lido.staking_router.get_staking_module_nonce(module_id) == old_module_nonce + 1 diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index 68a0d847..b09104eb 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -49,5 +49,5 @@ 'gas_price_calculator_integration', 'deposit_transaction_sender_integration', 'mellow_deposit_strategy', - 'mellow_deposit_strategy_integration' + 'mellow_deposit_strategy_integration', ] diff --git a/tests/fixtures/strategy.py b/tests/fixtures/strategy.py index d030983a..a6feadc5 100644 --- a/tests/fixtures/strategy.py +++ b/tests/fixtures/strategy.py @@ -25,12 +25,12 @@ def mellow_deposit_strategy_integration(web3_lido_integration): @pytest.fixture -def deposit_transaction_sender(web3_lido_unit): +def deposit_transaction_sender(web3_lido_unit) -> Sender: yield Sender(web3_lido_unit) @pytest.fixture -def deposit_transaction_sender_integration(web3_lido_integration): +def deposit_transaction_sender_integration(web3_lido_integration) -> Sender: yield Sender(web3_lido_integration)