From fc9a9c5ae2624bc26cf996560bedd19ed4ae93d9 Mon Sep 17 00:00:00 2001 From: F4ever <1590415904a@gmail.com> Date: Thu, 21 Sep 2023 11:02:12 +0200 Subject: [PATCH 1/5] from TRANSPORTS to MESSAGE_TRANSPORTS --- README.md | 4 ++-- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3d022cfe..b07f3b3a 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ This deposit is executed using the depositBufferedEther function within the "Dep - Set WEB3_RPC_ENDPOINTS - Set WALLET_PRIVATE_KEY - Set CREATE_TRANSACTIONS to true - - Set TRANSPORTS to rabbit + - Set MESSAGE_TRANSPORTS to rabbit - Set RABBIT_MQ_URL, RABBIT_MQ_USERNAME and RABBIT_MQ_PASSWORD 3. ```docker-compose up``` 4. Send metrics and logs to grafana @@ -49,7 +49,7 @@ This deposit is executed using the depositBufferedEther function within the "Dep | DEPOSIT_CONTRACT | 0x00000000219ab540356cBB839Cbe05303d7705Fa | Ethereum deposit contract address | | DEPOSIT_MODULES_WHITELIST | - | List of staking module's ids in which the depositor bot will make deposits | | --- | --- | --- | -| TRANSPORTS | - | Transports used in bot. One of/or both: rabbit/kafka | +| MESSAGE_TRANSPORTS | - | Transports used in bot. One of/or both: rabbit/kafka | | RABBIT_MQ_URL | - | RabbitMQ url | | RABBIT_MQ_USERNAME | - | RabbitMQ username for virtualhost | | RABBIT_MQ_PASSWORD | - | RabbitMQ password for virtualhost | diff --git a/docker-compose.yml b/docker-compose.yml index 9aa0c649..612f878e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - WEB3_RPC_ENDPOINTS=${WEB3_RPC_ENDPOINTS} - WALLET_PRIVATE_KEY=${WALLET_PRIVATE_KEY} - CREATE_TRANSACTIONS=${CREATE_TRANSACTIONS} - - TRANSPORTS=${TRANSPORTS} + - MESSAGE_TRANSPORTS=${MESSAGE_TRANSPORTS} - RABBIT_MQ_URL=${RABBIT_MQ_URL} - RABBIT_MQ_USERNAME=${RABBIT_MQ_USERNAME} - RABBIT_MQ_PASSWORD=${RABBIT_MQ_PASSWORD} @@ -31,7 +31,7 @@ services: - WEB3_RPC_ENDPOINTS=${WEB3_RPC_ENDPOINTS} - WALLET_PRIVATE_KEY=${WALLET_PRIVATE_KEY} - CREATE_TRANSACTIONS=${CREATE_TRANSACTIONS} - - TRANSPORTS=${TRANSPORTS} + - MESSAGE_TRANSPORTS=${MESSAGE_TRANSPORTS} - RABBIT_MQ_URL=${RABBIT_MQ_URL} - RABBIT_MQ_USERNAME=${RABBIT_MQ_USERNAME} - RABBIT_MQ_PASSWORD=${RABBIT_MQ_PASSWORD} From 0668580dde6d199418d34e4f4a3b3cbbdfde2f6f Mon Sep 17 00:00:00 2001 From: F4ever <1590415904a@gmail.com> Date: Thu, 21 Sep 2023 11:12:00 +0200 Subject: [PATCH 2/5] remove max check deposits --- README.md | 2 +- src/blockchain/contracts/deposit_security_module.py | 13 ------------- src/blockchain/deposit_strategy/curated_module.py | 7 +++---- .../deposit_strategy/prefered_module_to_deposit.py | 6 ++---- src/blockchain/{executer.py => executor.py} | 0 .../contracts/test_deposit_security_module.py | 3 +-- .../test_get_prefered_module_to_deposit.py | 10 ++++------ tests/bots/test_depositor.py | 2 -- 8 files changed, 11 insertions(+), 32 deletions(-) rename src/blockchain/{executer.py => executor.py} (100%) diff --git a/README.md b/README.md index b07f3b3a..83c6074c 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ This deposit is executed using the depositBufferedEther function within the "Dep | DEPOSIT_CONTRACT | 0x00000000219ab540356cBB839Cbe05303d7705Fa | Ethereum deposit contract address | | DEPOSIT_MODULES_WHITELIST | - | List of staking module's ids in which the depositor bot will make deposits | | --- | --- | --- | -| MESSAGE_TRANSPORTS | - | Transports used in bot. One of/or both: rabbit/kafka | +| MESSAGE_TRANSPORTS | - | Transports used in bot. One of/or both: rabbit/kafka | | RABBIT_MQ_URL | - | RabbitMQ url | | RABBIT_MQ_USERNAME | - | RabbitMQ username for virtualhost | | RABBIT_MQ_PASSWORD | - | RabbitMQ password for virtualhost | diff --git a/src/blockchain/contracts/deposit_security_module.py b/src/blockchain/contracts/deposit_security_module.py index a9cedbe3..ea4c1a30 100644 --- a/src/blockchain/contracts/deposit_security_module.py +++ b/src/blockchain/contracts/deposit_security_module.py @@ -5,7 +5,6 @@ from blockchain.contracts.base_interface import ContractInterface - logger = logging.getLogger(__name__) @@ -127,15 +126,3 @@ def pause_deposits( guardian_signature, )}) return tx - - def get_max_deposits(self, block_identifier: BlockIdentifier = 'latest'): - """ - Returns maxDepositsPerBlock - """ - response = self.functions.getMaxDeposits().call(block_identifier=block_identifier) - logger.info({ - 'msg': f'Call `getMaxDeposits()`.', - 'value': response, - 'block_identifier': block_identifier.__repr__(), - }) - return response diff --git a/src/blockchain/deposit_strategy/curated_module.py b/src/blockchain/deposit_strategy/curated_module.py index bf01bc2f..bb51a2e2 100644 --- a/src/blockchain/deposit_strategy/curated_module.py +++ b/src/blockchain/deposit_strategy/curated_module.py @@ -10,7 +10,6 @@ from blockchain.deposit_strategy.interface import ModuleDepositStrategyInterface from metrics.metrics import GAS_FEE, DEPOSITABLE_ETHER, POSSIBLE_DEPOSITS_AMOUNT - logger = logging.getLogger(__name__) @@ -61,7 +60,7 @@ def _calculate_recommended_gas_based_on_deposit_amount(self, deposits_amount: in GAS_FEE.labels('based_on_buffer_fee', self.module_id).set(recommended_max_gas) return recommended_max_gas - def _get_pending_base_fee(self): + def _get_pending_base_fee(self) -> Wei: base_fee_per_gas = self.w3.eth.get_block('pending')['baseFeePerGas'] logger.info({'msg': 'Fetch base_fee_per_gas for pending block.', 'value': base_fee_per_gas}) return base_fee_per_gas @@ -81,9 +80,9 @@ def is_gas_price_ok(self) -> bool: return recommended_gas_fee >= current_gas_fee - def _get_recommended_gas_fee(self): + def _get_recommended_gas_fee(self) -> Wei: gas_history = self._fetch_gas_fee_history(variables.GAS_FEE_PERCENTILE_DAYS_HISTORY_1) - return int(numpy.percentile(gas_history, variables.GAS_FEE_PERCENTILE_1)) + return Wei(int(numpy.percentile(gas_history, variables.GAS_FEE_PERCENTILE_1))) def _fetch_gas_fee_history(self, days: int) -> list[int]: latest_block_num = self.w3.eth.get_block('latest')['number'] diff --git a/src/blockchain/deposit_strategy/prefered_module_to_deposit.py b/src/blockchain/deposit_strategy/prefered_module_to_deposit.py index 4d48ff46..82118816 100644 --- a/src/blockchain/deposit_strategy/prefered_module_to_deposit.py +++ b/src/blockchain/deposit_strategy/prefered_module_to_deposit.py @@ -18,7 +18,7 @@ def get_preferred_to_deposit_module(w3: Web3, whitelist_modules: list[int]) -> O stats = get_modules_stats(w3, active_modules) # Return module id - return stats[0][2] + return stats[0][1] def get_active_modules(w3: Web3, whitelist_modules: list[int]) -> list[int]: @@ -34,13 +34,11 @@ def get_active_modules(w3: Web3, whitelist_modules: list[int]) -> list[int]: return modules -def get_modules_stats(w3: Web3, modules: list[int]) -> list[tuple[int, int, int]]: - max_deposits_count = w3.lido.deposit_security_module.get_max_deposits() +def get_modules_stats(w3: Web3, modules: list[int]) -> list[tuple[int, int]]: depositable_ether = w3.lido.lido.get_depositable_ether() module_stats = [( w3.lido.staking_router.get_staking_module_max_deposits_count(module, depositable_ether), - w3.lido.staking_router.get_staking_module_max_deposits_count(module, max_deposits_count * 32 * 10 ** 18), module, ) for module in modules ] diff --git a/src/blockchain/executer.py b/src/blockchain/executor.py similarity index 100% rename from src/blockchain/executer.py rename to src/blockchain/executor.py diff --git a/tests/blockchain/contracts/test_deposit_security_module.py b/tests/blockchain/contracts/test_deposit_security_module.py index 7509d192..9e329945 100644 --- a/tests/blockchain/contracts/test_deposit_security_module.py +++ b/tests/blockchain/contracts/test_deposit_security_module.py @@ -1,7 +1,7 @@ import pytest from tests.utils.contract_utils import check_contract -from tests.utils.regrex import HASH_REGREX, check_value_re, check_value_type, ADDRESS_REGREX +from tests.utils.regrex import check_value_re, check_value_type, ADDRESS_REGREX @pytest.mark.integration @@ -16,7 +16,6 @@ def test_deposit_security_module_call(deposit_security_module, caplog): ('can_deposit', (1,), lambda response: check_value_type(response, bool)), ('get_pause_message_prefix', None, lambda response: check_value_type(response, bytes)), ('get_pause_intent_validity_period_blocks', None, lambda response: check_value_type(response, int)), - ('get_max_deposits', None, lambda response: check_value_type(response, int)), ], caplog, ) diff --git a/tests/blockchain/deposit_strategy/test_get_prefered_module_to_deposit.py b/tests/blockchain/deposit_strategy/test_get_prefered_module_to_deposit.py index 2c0023d5..cc9eb5aa 100644 --- a/tests/blockchain/deposit_strategy/test_get_prefered_module_to_deposit.py +++ b/tests/blockchain/deposit_strategy/test_get_prefered_module_to_deposit.py @@ -13,7 +13,6 @@ def test_get_preferred_to_deposit_module(web3_lido_unit): modules = list(range(10)) - web3_lido_unit.lido.deposit_security_module.get_max_deposits = Mock(return_value=100) web3_lido_unit.lido.lido.get_depositable_ether = Mock(return_value=10 * 32 * 10 ** 18) web3_lido_unit.lido.staking_router.get_staking_module_ids = Mock(return_value=modules) web3_lido_unit.lido.staking_router.get_staking_module_max_deposits_count = Mock(return_value=0) @@ -23,6 +22,7 @@ def test_get_preferred_to_deposit_module(web3_lido_unit): assert result == 7 +@pytest.mark.unit def test_active_modules(web3_lido_unit): web3_lido_unit.lido.staking_router.get_staking_module_ids = Mock(return_value=[1, 2, 3, 4, 5, 6]) web3_lido_unit.lido.staking_router.is_staking_module_active = lambda x: x % 2 @@ -32,8 +32,8 @@ def test_active_modules(web3_lido_unit): assert modules_list == [1, 3] +@pytest.mark.unit def test_get_module_stats(web3_lido_unit): - web3_lido_unit.lido.deposit_security_module.get_max_deposits = Mock(return_value=100) web3_lido_unit.lido.lido.get_depositable_ether = Mock(return_value=10 * 32 * 10 ** 18) web3_lido_unit.lido.staking_router.get_staking_module_max_deposits_count = lambda x, y: x % 3 @@ -42,7 +42,5 @@ def test_get_module_stats(web3_lido_unit): for i in range(len(stats) - 1): assert stats[i][0] >= stats[i + 1][0] - if stats[i][1] > stats[i + 1][1]: - assert stats[i][2] <= stats[i - 1][2] - elif stats[i][1] == stats[i + 1][1]: - assert stats[i][0] == stats[i + 1][0] + if stats[i][0] == stats[i + 1][0]: + assert stats[i + 1][1] < stats[i][1] diff --git a/tests/bots/test_depositor.py b/tests/bots/test_depositor.py index 094abfd7..5b793e57 100644 --- a/tests/bots/test_depositor.py +++ b/tests/bots/test_depositor.py @@ -7,7 +7,6 @@ from bots.depositor import DepositorBot from tests.conftest import DSM_OWNER - COUNCIL_ADDRESS_1 = '0x70997970C51812dc3A010C7d01b50e0d17dc79C8' COUNCIL_PK_1 = '0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d' @@ -53,7 +52,6 @@ def deposit_message(): def test_depositor_one_module_deposited(depositor_bot, block_data): modules = list(range(10)) - depositor_bot.w3.lido.deposit_security_module.get_max_deposits = Mock(return_value=100) depositor_bot.w3.lido.lido.get_depositable_ether = Mock(return_value=10 * 32 * 10 ** 18) depositor_bot.w3.lido.staking_router.get_staking_module_ids = Mock(return_value=modules) depositor_bot.w3.lido.staking_router.get_staking_module_max_deposits_count = Mock(return_value=0) From 455325cd3782c63b4d8c257018dc02c57c159446 Mon Sep 17 00:00:00 2001 From: F4ever <1590415904a@gmail.com> Date: Thu, 21 Sep 2023 11:47:58 +0200 Subject: [PATCH 3/5] review fixes --- README.md | 2 +- src/blockchain/web3_extentions/transaction.py | 7 +++---- src/bots/depositor.py | 10 +++++++--- src/bots/{pause.py => pauser.py} | 0 src/depositor.py | 9 ++++----- src/metrics/metrics.py | 11 +++++++++-- src/metrics/transport_message_metrics.py | 5 ++--- src/pauser.py | 9 ++++----- src/transport/msg_schemas.py | 12 +++++++----- tests/bots/test_depositor.py | 12 ++++++++++++ 10 files changed, 49 insertions(+), 28 deletions(-) rename src/bots/{pause.py => pauser.py} (100%) diff --git a/README.md b/README.md index 83c6074c..97414f21 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ This deposit is executed using the depositBufferedEther function within the "Dep | CREATE_TRANSACTIONS | false | If true then tx will be send to blockchain | | LIDO_LOCATOR | 0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb | Lido Locator address. Mainnet by default. Other networks could be found [here](https://docs.lido.fi/deployed-contracts/) | | DEPOSIT_CONTRACT | 0x00000000219ab540356cBB839Cbe05303d7705Fa | Ethereum deposit contract address | -| DEPOSIT_MODULES_WHITELIST | - | List of staking module's ids in which the depositor bot will make deposits | +| DEPOSIT_MODULES_WHITELIST | 1 | List of staking module's ids in which the depositor bot will make deposits | | --- | --- | --- | | MESSAGE_TRANSPORTS | - | Transports used in bot. One of/or both: rabbit/kafka | | RABBIT_MQ_URL | - | RabbitMQ url | diff --git a/src/blockchain/web3_extentions/transaction.py b/src/blockchain/web3_extentions/transaction.py index 62c3300e..bafad7fa 100644 --- a/src/blockchain/web3_extentions/transaction.py +++ b/src/blockchain/web3_extentions/transaction.py @@ -10,7 +10,6 @@ from blockchain.constants import SLOT_TIME from metrics.metrics import TX_SEND - logger = logging.getLogger(__name__) @@ -92,7 +91,7 @@ def flashbots_send( except TransactionNotFound: return False else: - logger.info({'msg': 'Sent transaction found.', 'value': rec[-1]['transactionHash'].hex()}) + logger.info({'msg': 'Sent transaction included in blockchain.', 'value': rec[-1]['transactionHash'].hex()}) return True def classic_send(self, signed_tx: SignedTransaction, timeout_in_blocks: int) -> bool: @@ -102,13 +101,13 @@ def classic_send(self, signed_tx: SignedTransaction, timeout_in_blocks: int) -> logger.error({'msg': 'Transaction reverted.', 'value': str(error)}) return False - logger.info({'msg': 'Sent transaction found.', 'value': tx_hash.hex()}) + logger.info({'msg': 'Transaction sent.', 'value': tx_hash.hex()}) try: tx_receipt = self.web3.eth.wait_for_transaction_receipt(tx_hash, (timeout_in_blocks + 1) * SLOT_TIME) except TimeExhausted: return False - logger.info({'msg': 'Transaction found', 'value': tx_receipt['transactionHash'].hex()}) + logger.info({'msg': 'Sent transaction included in blockchain.', 'value': tx_receipt['transactionHash'].hex()}) return True def _get_priority_fee(self, percentile: int, min_priority_fee: Wei, max_priority_fee: Wei): diff --git a/src/bots/depositor.py b/src/bots/depositor.py index 3a73516c..c01c4f9e 100644 --- a/src/bots/depositor.py +++ b/src/bots/depositor.py @@ -7,14 +7,15 @@ from web3.types import BlockData import variables - from blockchain.deposit_strategy.curated_module import CuratedModuleDepositStrategy from blockchain.deposit_strategy.interface import ModuleDepositStrategyInterface from blockchain.deposit_strategy.prefered_module_to_deposit import get_preferred_to_deposit_module from blockchain.typings import Web3 from cryptography.verify_signature import compute_vs from metrics.metrics import ( - ACCOUNT_BALANCE, CURRENT_QUORUM_SIZE, + ACCOUNT_BALANCE, + CURRENT_QUORUM_SIZE, + UNEXPECTED_EXCEPTIONS, ) from metrics.transport_message_metrics import message_metrics_filter from transport.msg_providers.kafka import KafkaMessageProvider @@ -29,7 +30,6 @@ from transport.msg_storage import MessageStorage from transport.types import TransportType - logger = logging.getLogger(__name__) @@ -165,6 +165,10 @@ def _get_message_actualize_filter(self, module_id: int) -> Callable[[DepositMess def message_filter(message: DepositMessage) -> bool: if message['guardianAddress'] not in guardians_list: + UNEXPECTED_EXCEPTIONS.labels('unexpected_guardian_address').inc() + return False + + if message['stakingModuleId'] != module_id: return False if message['blockNumber'] < latest['number'] - 200: diff --git a/src/bots/pause.py b/src/bots/pauser.py similarity index 100% rename from src/bots/pause.py rename to src/bots/pauser.py diff --git a/src/depositor.py b/src/depositor.py index d2d617a5..bf3892e0 100644 --- a/src/depositor.py +++ b/src/depositor.py @@ -1,19 +1,18 @@ -from prometheus_client import start_http_server +from blockchain.executer import Executor from flashbots import flashbot +from prometheus_client import start_http_server from web3 import Web3 from web3_multi_provider import FallbackProvider import variables -from blockchain.executer import Executor from blockchain.web3_extentions.lido_contracts import LidoContracts +from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from blockchain.web3_extentions.transaction import TransactionUtils from bots.depositor import DepositorBot from metrics.healthcheck_pulse import start_pulse_server from metrics.logging import logging -from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from metrics.metrics import BUILD_INFO - logger = logging.getLogger(__name__) @@ -67,7 +66,7 @@ def main(): 5, variables.MAX_CYCLE_LIFETIME_IN_SECONDS, ) - logger.info({'msg': 'Rum executor.'}) + logger.info({'msg': 'Execute depositor as daemon.'}) e.execute_as_daemon() diff --git a/src/metrics/metrics.py b/src/metrics/metrics.py index 234b4a63..183947a4 100644 --- a/src/metrics/metrics.py +++ b/src/metrics/metrics.py @@ -26,13 +26,13 @@ DEPOSIT_MESSAGES = Gauge( 'deposit_messages', 'Guardians deposit messages', - ['address', 'version'], + ['address', 'module_id', 'version'], namespace=PREFIX, ) PAUSE_MESSAGES = Gauge( 'pause_messages', 'Guardians pause messages', - ['address', 'version'], + ['address', 'module_id', 'version'], namespace=PREFIX, ) PING_MESSAGES = Gauge( @@ -74,3 +74,10 @@ ['method', 'code', 'domain'], namespace=PREFIX ) + +UNEXPECTED_EXCEPTIONS = Counter( + 'unexpected_exceptions', + 'Total count of unexpected exceptions', + ['type'], + namespace=PREFIX, +) diff --git a/src/metrics/transport_message_metrics.py b/src/metrics/transport_message_metrics.py index 2e7baa40..1e1f3e2e 100644 --- a/src/metrics/transport_message_metrics.py +++ b/src/metrics/transport_message_metrics.py @@ -4,7 +4,6 @@ from transport.msg_providers.rabbit import MessageType from transport.msg_schemas import DepositMessage - logger = logging.getLogger(__name__) @@ -15,11 +14,11 @@ def message_metrics_filter(msg: DepositMessage) -> bool: address, version = msg.get('guardianAddress'), msg.get('app', {}).get('version') if msg_type == MessageType.PAUSE: - PAUSE_MESSAGES.labels(address, version).inc() + PAUSE_MESSAGES.labels(address, msg.get('stakingModuleId', -1), version).inc() return True if msg_type == MessageType.DEPOSIT: - DEPOSIT_MESSAGES.labels(address, version).inc() + DEPOSIT_MESSAGES.labels(address, msg.get('stakingModuleId', -1), version).inc() return True elif msg_type == MessageType.PING: diff --git a/src/pauser.py b/src/pauser.py index d86b82fe..619a230f 100644 --- a/src/pauser.py +++ b/src/pauser.py @@ -1,18 +1,17 @@ +from blockchain.executer import Executor +from bots.pause import PauserBot from prometheus_client import start_http_server from web3 import Web3 from web3_multi_provider import FallbackProvider import variables -from blockchain.executer import Executor from blockchain.web3_extentions.lido_contracts import LidoContracts +from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from blockchain.web3_extentions.transaction import TransactionUtils -from bots.pause import PauserBot from metrics.healthcheck_pulse import start_pulse_server from metrics.logging import logging -from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from metrics.metrics import BUILD_INFO - logger = logging.getLogger(__name__) @@ -60,7 +59,7 @@ def main(): 1, variables.MAX_CYCLE_LIFETIME_IN_SECONDS, ) - logger.info({'msg': 'Rum executor.'}) + logger.info({'msg': 'Execute depositor as daemon.'}) e.execute_as_daemon() diff --git a/src/transport/msg_schemas.py b/src/transport/msg_schemas.py index 54db33cd..d5ce6b82 100644 --- a/src/transport/msg_schemas.py +++ b/src/transport/msg_schemas.py @@ -6,7 +6,7 @@ from web3 import Web3 from cryptography.verify_signature import verify_message_with_signature - +from metrics.metrics import UNEXPECTED_EXCEPTIONS logger = logging.getLogger(__name__) @@ -74,11 +74,11 @@ class DepositMessage(TypedDict): app: dict -def get_deposit_messages_sign_filter(attestation_prefix: bytes) -> Callable: +def get_deposit_messages_sign_filter(deposit_prefix: bytes) -> Callable: """Returns filter that checks message validity""" def check_deposit_messages(msg: DepositMessage) -> bool: verified = verify_message_with_signature( - data=[attestation_prefix, msg['blockNumber'], msg['blockHash'], msg['depositRoot'], msg['stakingModuleId'], msg['nonce']], + data=[deposit_prefix, msg['blockNumber'], msg['blockHash'], msg['depositRoot'], msg['stakingModuleId'], msg['nonce']], abi=['bytes32', 'uint256', 'bytes32', 'bytes32', 'uint256', 'uint256'], address=msg['guardianAddress'], vrs=( @@ -90,6 +90,7 @@ def check_deposit_messages(msg: DepositMessage) -> bool: if not verified: logger.error({'msg': 'Message verification failed.', 'value': msg}) + UNEXPECTED_EXCEPTIONS.labels('deposit_message_verification_failed').inc() return verified @@ -130,10 +131,10 @@ class PauseMessage(TypedDict): stakingModuleId: int -def get_pause_messages_sign_filter(attestation_prefix: bytes) -> Callable: +def get_pause_messages_sign_filter(pause_prefix: bytes) -> Callable: def check_pause_message(msg: PauseMessage) -> bool: verified = verify_message_with_signature( - data=[attestation_prefix, msg['blockNumber'], msg['stakingModuleId']], + data=[pause_prefix, msg['blockNumber'], msg['stakingModuleId']], abi=['bytes32', 'uint256', 'uint256'], address=msg['guardianAddress'], vrs=( @@ -145,6 +146,7 @@ def check_pause_message(msg: PauseMessage) -> bool: if not verified: logger.error({'msg': 'Message verification failed.', 'value': msg}) + UNEXPECTED_EXCEPTIONS.labels('pause_message_verification_failed').inc() return verified diff --git a/tests/bots/test_depositor.py b/tests/bots/test_depositor.py index 5b793e57..7d3da621 100644 --- a/tests/bots/test_depositor.py +++ b/tests/bots/test_depositor.py @@ -135,6 +135,18 @@ def test_depositor_message_actualizer_not_guardian(setup_deposit_message, deposi assert not list(filter(message_filter, [deposit_message])) +@pytest.mark.unit +def test_depositor_message_actualizer_no_selected_module(setup_deposit_message, depositor_bot, deposit_message, block_data): + second = deposit_message.copy() + second['stakingModuleId'] = 2 + + message_filter = depositor_bot._get_message_actualize_filter(2) + assert not list(filter(message_filter, [ + deposit_message, + ])) + assert len(list(filter(message_filter, [deposit_message, second]))) == 1 + + @pytest.mark.unit def test_depositor_message_actualizer_outdated(setup_deposit_message, depositor_bot, deposit_message, block_data): deposit_message['blockNumber'] = block_data['number'] - 250 From e4fe57793a495605992839d75867642ef49c85d4 Mon Sep 17 00:00:00 2001 From: F4ever <1590415904a@gmail.com> Date: Thu, 21 Sep 2023 11:56:09 +0200 Subject: [PATCH 4/5] fix imports --- src/depositor.py | 2 +- src/pauser.py | 4 ++-- tests/bots/test_executor.py | 2 +- tests/bots/test_pauser.py | 5 +---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/depositor.py b/src/depositor.py index bf3892e0..50b1d491 100644 --- a/src/depositor.py +++ b/src/depositor.py @@ -1,10 +1,10 @@ -from blockchain.executer import Executor from flashbots import flashbot from prometheus_client import start_http_server from web3 import Web3 from web3_multi_provider import FallbackProvider import variables +from blockchain.executor import Executor from blockchain.web3_extentions.lido_contracts import LidoContracts from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from blockchain.web3_extentions.transaction import TransactionUtils diff --git a/src/pauser.py b/src/pauser.py index 619a230f..da367e32 100644 --- a/src/pauser.py +++ b/src/pauser.py @@ -1,13 +1,13 @@ -from blockchain.executer import Executor -from bots.pause import PauserBot from prometheus_client import start_http_server from web3 import Web3 from web3_multi_provider import FallbackProvider import variables +from blockchain.executor import Executor from blockchain.web3_extentions.lido_contracts import LidoContracts from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware from blockchain.web3_extentions.transaction import TransactionUtils +from bots.pauser import PauserBot from metrics.healthcheck_pulse import start_pulse_server from metrics.logging import logging from metrics.metrics import BUILD_INFO diff --git a/tests/bots/test_executor.py b/tests/bots/test_executor.py index 80f71f8a..35970320 100644 --- a/tests/bots/test_executor.py +++ b/tests/bots/test_executor.py @@ -4,7 +4,7 @@ import pytest from web3.types import BlockData -from blockchain.executer import Executor +from blockchain.executor import Executor from metrics import healthcheck_pulse from utils.timeout import TimeoutManagerError diff --git a/tests/bots/test_pauser.py b/tests/bots/test_pauser.py index c6b4e55a..e2e57ae0 100644 --- a/tests/bots/test_pauser.py +++ b/tests/bots/test_pauser.py @@ -1,12 +1,9 @@ from unittest.mock import Mock import pytest -from eth_abi import encode -from eth_account.messages import encode_defunct, encode_structured_data -from eth_hash.backends.pycryptodome import keccak256 import variables -from bots.pause import PauserBot +from bots.pauser import PauserBot from tests.conftest import DSM_OWNER # WARNING: These accounts, and their private keys, are publicly known. From 2178ff09ddbca01a931332b50926ea643d2d723b Mon Sep 17 00:00:00 2001 From: F4ever <1590415904a@gmail.com> Date: Thu, 21 Sep 2023 13:54:14 +0200 Subject: [PATCH 5/5] add gas estimation --- src/blockchain/web3_extentions/transaction.py | 21 +++++++++++++++- .../blockchain/web3_extentions/transaction.py | 25 +++++++++++++++++++ tests/fixtures/provider.py | 5 +++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/blockchain/web3_extentions/transaction.py b/src/blockchain/web3_extentions/transaction.py index bafad7fa..c0afad22 100644 --- a/src/blockchain/web3_extentions/transaction.py +++ b/src/blockchain/web3_extentions/transaction.py @@ -1,6 +1,7 @@ import logging from eth_account.datastructures import SignedTransaction +from eth_typing import ChecksumAddress from web3.contract import ContractFunction from web3.exceptions import ContractLogicError, TransactionNotFound, TimeExhausted from web3.module import Module @@ -47,10 +48,12 @@ def send( variables.MAX_PRIORITY_FEE, ) + gas_limit = self._estimate_gas(transaction, variables.ACCOUNT.address) + transaction_dict = transaction.build_transaction({ 'from': variables.ACCOUNT.address, # TODO Estimate gas and min(contract_gas_limit, estimated_gas * 1.3) - 'gas': variables.CONTRACT_GAS_LIMIT, + 'gas': gas_limit, 'maxFeePerGas': pending['baseFeePerGas'] * 2 + priority, 'maxPriorityFeePerGas': priority, "nonce": self.web3.eth.get_transaction_count(variables.ACCOUNT.address), @@ -73,6 +76,22 @@ def send( return status + @staticmethod + def _estimate_gas(transaction: ContractFunction, account_address: ChecksumAddress) -> int: + try: + gas = transaction.estimate_gas({'from': account_address}) + except ContractLogicError as error: + logger.warning({'msg': 'Can not estimate gas. Contract logic error.', 'error': str(error)}) + return variables.CONTRACT_GAS_LIMIT + except ValueError as error: + logger.warning({'msg': 'Can not estimate gas. Execution reverted.', 'error': str(error)}) + return variables.CONTRACT_GAS_LIMIT + + return min( + variables.CONTRACT_GAS_LIMIT, + int(gas * 1.3), + ) + def flashbots_send( self, signed_tx: SignedTransaction, diff --git a/tests/blockchain/web3_extentions/transaction.py b/tests/blockchain/web3_extentions/transaction.py index f12acfee..5a9c27ae 100644 --- a/tests/blockchain/web3_extentions/transaction.py +++ b/tests/blockchain/web3_extentions/transaction.py @@ -2,6 +2,7 @@ from unittest.mock import Mock import pytest +from web3.exceptions import ContractLogicError import variables from blockchain.web3_extentions.transaction import TransactionUtils @@ -35,3 +36,27 @@ def test_protector_create_tx(web3_lido_unit, set_integration_account, caplog): variables.CREATE_TRANSACTIONS = False tu.send(None, False, 10) assert 'Dry mode activated. Sending transaction skipped.' in caplog.messages[-1] + + +class Transaction: + args = {} + + def estimate_gas(self, params: dict) -> int: + return 0 + + +@pytest.mark.unit +def test_estimate_gas(web3_lido_unit, set_account): + tx = Transaction() + + tx.estimate_gas = Mock(return_value=variables.CONTRACT_GAS_LIMIT * 2) + gas_amount = web3_lido_unit.transaction._estimate_gas(tx, variables.ACCOUNT.address) + assert gas_amount == variables.CONTRACT_GAS_LIMIT + + tx.estimate_gas = Mock(return_value=100) + gas_amount = web3_lido_unit.transaction._estimate_gas(tx, variables.ACCOUNT.address) + assert gas_amount == 130 + + tx.estimate_gas = Mock(side_effect=ContractLogicError()) + gas_amount = web3_lido_unit.transaction._estimate_gas(tx, variables.ACCOUNT.address) + assert gas_amount == variables.CONTRACT_GAS_LIMIT diff --git a/tests/fixtures/provider.py b/tests/fixtures/provider.py index 80ae4eb3..8c28a36b 100644 --- a/tests/fixtures/provider.py +++ b/tests/fixtures/provider.py @@ -14,7 +14,10 @@ def web3_lido_unit(): web3 = Web3() web3.lido = Mock() - web3.transaction = Mock() + web3.attach_modules({ + 'transaction': TransactionUtils, + }) + yield web3