Merge pull request #176 from lidofinance/feat/gas-estimation

Feat/gas estimation

README.md

@@ -30,7 +30,7 @@ This deposit is executed using the depositBufferedEther function within the "Dep
    - Set WEB3_RPC_ENDPOINTS
    - Set WALLET_PRIVATE_KEY
    - Set CREATE_TRANSACTIONS to true
-   - Set TRANSPORTS to rabbit
+   - Set MESSAGE_TRANSPORTS to rabbit
    - Set RABBIT_MQ_URL, RABBIT_MQ_USERNAME and RABBIT_MQ_PASSWORD
 3. ```docker-compose up```
 4. Send metrics and logs to grafana
@@ -47,9 +47,9 @@ This deposit is executed using the depositBufferedEther function within the "Dep
 | CREATE_TRANSACTIONS       | false                                      | If true then tx will be send to blockchain                                                                               |
 | LIDO_LOCATOR              | 0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb | Lido Locator address. Mainnet by default. Other networks could be found [here](https://docs.lido.fi/deployed-contracts/) |
 | DEPOSIT_CONTRACT          | 0x00000000219ab540356cBB839Cbe05303d7705Fa | Ethereum deposit contract address                                                                                        |
-| DEPOSIT_MODULES_WHITELIST | -                                          | List of staking module's ids in which the depositor bot will make deposits                                               |
+| DEPOSIT_MODULES_WHITELIST | 1                                          | List of staking module's ids in which the depositor bot will make deposits                                               |
 | ---                       | ---	                                       | ---                                                                                                                      |
-| TRANSPORTS                | -                                          | Transports used in bot. One of/or both: rabbit/kafka                                                                     |
+| MESSAGE_TRANSPORTS        | -                                          | Transports used in bot. One of/or both: rabbit/kafka                                                                     |
 | RABBIT_MQ_URL             | -                                          | RabbitMQ url                                                                                                             |
 | RABBIT_MQ_USERNAME        | -                                          | RabbitMQ username for virtualhost                                                                                        |
 | RABBIT_MQ_PASSWORD        | -                                          | RabbitMQ password for virtualhost                                                                                        |

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - WEB3_RPC_ENDPOINTS=${WEB3_RPC_ENDPOINTS}
       - WALLET_PRIVATE_KEY=${WALLET_PRIVATE_KEY}
       - CREATE_TRANSACTIONS=${CREATE_TRANSACTIONS}
-      - TRANSPORTS=${TRANSPORTS}
+      - MESSAGE_TRANSPORTS=${MESSAGE_TRANSPORTS}
       - RABBIT_MQ_URL=${RABBIT_MQ_URL}
       - RABBIT_MQ_USERNAME=${RABBIT_MQ_USERNAME}
       - RABBIT_MQ_PASSWORD=${RABBIT_MQ_PASSWORD}
@@ -31,7 +31,7 @@ services:
       - WEB3_RPC_ENDPOINTS=${WEB3_RPC_ENDPOINTS}
       - WALLET_PRIVATE_KEY=${WALLET_PRIVATE_KEY}
       - CREATE_TRANSACTIONS=${CREATE_TRANSACTIONS}
-      - TRANSPORTS=${TRANSPORTS}
+      - MESSAGE_TRANSPORTS=${MESSAGE_TRANSPORTS}
       - RABBIT_MQ_URL=${RABBIT_MQ_URL}
       - RABBIT_MQ_USERNAME=${RABBIT_MQ_USERNAME}
       - RABBIT_MQ_PASSWORD=${RABBIT_MQ_PASSWORD}

src/blockchain/contracts/deposit_security_module.py

@@ -5,7 +5,6 @@
 
 from blockchain.contracts.base_interface import ContractInterface
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -127,15 +126,3 @@ def pause_deposits(
             guardian_signature,
         )})
         return tx
-
-    def get_max_deposits(self, block_identifier: BlockIdentifier = 'latest'):
-        """
-        Returns maxDepositsPerBlock
-        """
-        response = self.functions.getMaxDeposits().call(block_identifier=block_identifier)
-        logger.info({
-            'msg': f'Call `getMaxDeposits()`.',
-            'value': response,
-            'block_identifier': block_identifier.__repr__(),
-        })
-        return response

src/blockchain/deposit_strategy/curated_module.py

@@ -10,7 +10,6 @@
 from blockchain.deposit_strategy.interface import ModuleDepositStrategyInterface
 from metrics.metrics import GAS_FEE, DEPOSITABLE_ETHER, POSSIBLE_DEPOSITS_AMOUNT
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -61,7 +60,7 @@ def _calculate_recommended_gas_based_on_deposit_amount(self, deposits_amount: in
         GAS_FEE.labels('based_on_buffer_fee', self.module_id).set(recommended_max_gas)
         return recommended_max_gas
 
-    def _get_pending_base_fee(self):
+    def _get_pending_base_fee(self) -> Wei:
         base_fee_per_gas = self.w3.eth.get_block('pending')['baseFeePerGas']
         logger.info({'msg': 'Fetch base_fee_per_gas for pending block.', 'value': base_fee_per_gas})
         return base_fee_per_gas
@@ -81,9 +80,9 @@ def is_gas_price_ok(self) -> bool:
 
         return recommended_gas_fee >= current_gas_fee
 
-    def _get_recommended_gas_fee(self):
+    def _get_recommended_gas_fee(self) -> Wei:
         gas_history = self._fetch_gas_fee_history(variables.GAS_FEE_PERCENTILE_DAYS_HISTORY_1)
-        return int(numpy.percentile(gas_history, variables.GAS_FEE_PERCENTILE_1))
+        return Wei(int(numpy.percentile(gas_history, variables.GAS_FEE_PERCENTILE_1)))
 
     def _fetch_gas_fee_history(self, days: int) -> list[int]:
         latest_block_num = self.w3.eth.get_block('latest')['number']

src/blockchain/deposit_strategy/prefered_module_to_deposit.py

@@ -18,7 +18,7 @@ def get_preferred_to_deposit_module(w3: Web3, whitelist_modules: list[int]) -> O
     stats = get_modules_stats(w3, active_modules)
 
     # Return module id
-    return stats[0][2]
+    return stats[0][1]
 
 
 def get_active_modules(w3: Web3, whitelist_modules: list[int]) -> list[int]:
@@ -34,13 +34,11 @@ def get_active_modules(w3: Web3, whitelist_modules: list[int]) -> list[int]:
     return modules
 
 
-def get_modules_stats(w3: Web3, modules: list[int]) -> list[tuple[int, int, int]]:
-    max_deposits_count = w3.lido.deposit_security_module.get_max_deposits()
+def get_modules_stats(w3: Web3, modules: list[int]) -> list[tuple[int, int]]:
     depositable_ether = w3.lido.lido.get_depositable_ether()
 
     module_stats = [(
             w3.lido.staking_router.get_staking_module_max_deposits_count(module, depositable_ether),
-            w3.lido.staking_router.get_staking_module_max_deposits_count(module, max_deposits_count * 32 * 10 ** 18),
             module,
         ) for module in modules
     ]

src/blockchain/web3_extentions/transaction.py

@@ -1,6 +1,7 @@
 import logging
 
 from eth_account.datastructures import SignedTransaction
+from eth_typing import ChecksumAddress
 from web3.contract import ContractFunction
 from web3.exceptions import ContractLogicError, TransactionNotFound, TimeExhausted
 from web3.module import Module
@@ -10,7 +11,6 @@
 from blockchain.constants import SLOT_TIME
 from metrics.metrics import TX_SEND
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -48,10 +48,12 @@ def send(
             variables.MAX_PRIORITY_FEE,
         )
 
+        gas_limit = self._estimate_gas(transaction, variables.ACCOUNT.address)
+
         transaction_dict = transaction.build_transaction({
             'from': variables.ACCOUNT.address,
             # TODO Estimate gas and min(contract_gas_limit, estimated_gas * 1.3)
-            'gas': variables.CONTRACT_GAS_LIMIT,
+            'gas': gas_limit,
             'maxFeePerGas': pending['baseFeePerGas'] * 2 + priority,
             'maxPriorityFeePerGas': priority,
             "nonce": self.web3.eth.get_transaction_count(variables.ACCOUNT.address),
@@ -74,6 +76,22 @@ def send(
 
         return status
 
+    @staticmethod
+    def _estimate_gas(transaction: ContractFunction, account_address: ChecksumAddress) -> int:
+        try:
+            gas = transaction.estimate_gas({'from': account_address})
+        except ContractLogicError as error:
+            logger.warning({'msg': 'Can not estimate gas. Contract logic error.', 'error': str(error)})
+            return variables.CONTRACT_GAS_LIMIT
+        except ValueError as error:
+            logger.warning({'msg': 'Can not estimate gas. Execution reverted.', 'error': str(error)})
+            return variables.CONTRACT_GAS_LIMIT
+
+        return min(
+            variables.CONTRACT_GAS_LIMIT,
+            int(gas * 1.3),
+        )
+
     def flashbots_send(
         self,
         signed_tx: SignedTransaction,
@@ -92,7 +110,7 @@ def flashbots_send(
         except TransactionNotFound:
             return False
         else:
-            logger.info({'msg': 'Sent transaction found.', 'value': rec[-1]['transactionHash'].hex()})
+            logger.info({'msg': 'Sent transaction included in blockchain.', 'value': rec[-1]['transactionHash'].hex()})
             return True
 
     def classic_send(self, signed_tx: SignedTransaction, timeout_in_blocks: int) -> bool:
@@ -102,13 +120,13 @@ def classic_send(self, signed_tx: SignedTransaction, timeout_in_blocks: int) ->
             logger.error({'msg': 'Transaction reverted.', 'value': str(error)})
             return False
 
-        logger.info({'msg': 'Sent transaction found.', 'value': tx_hash.hex()})
+        logger.info({'msg': 'Transaction sent.', 'value': tx_hash.hex()})
         try:
             tx_receipt = self.web3.eth.wait_for_transaction_receipt(tx_hash, (timeout_in_blocks + 1) * SLOT_TIME)
         except TimeExhausted:
             return False
 
-        logger.info({'msg': 'Transaction found', 'value': tx_receipt['transactionHash'].hex()})
+        logger.info({'msg': 'Sent transaction included in blockchain.', 'value': tx_receipt['transactionHash'].hex()})
         return True
 
     def _get_priority_fee(self, percentile: int, min_priority_fee: Wei, max_priority_fee: Wei):

src/bots/depositor.py

@@ -7,14 +7,15 @@
 from web3.types import BlockData
 
 import variables
-
 from blockchain.deposit_strategy.curated_module import CuratedModuleDepositStrategy
 from blockchain.deposit_strategy.interface import ModuleDepositStrategyInterface
 from blockchain.deposit_strategy.prefered_module_to_deposit import get_preferred_to_deposit_module
 from blockchain.typings import Web3
 from cryptography.verify_signature import compute_vs
 from metrics.metrics import (
-    ACCOUNT_BALANCE, CURRENT_QUORUM_SIZE,
+    ACCOUNT_BALANCE,
+    CURRENT_QUORUM_SIZE,
+    UNEXPECTED_EXCEPTIONS,
 )
 from metrics.transport_message_metrics import message_metrics_filter
 from transport.msg_providers.kafka import KafkaMessageProvider
@@ -29,7 +30,6 @@
 from transport.msg_storage import MessageStorage
 from transport.types import TransportType
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -165,6 +165,10 @@ def _get_message_actualize_filter(self, module_id: int) -> Callable[[DepositMess
 
         def message_filter(message: DepositMessage) -> bool:
             if message['guardianAddress'] not in guardians_list:
+                UNEXPECTED_EXCEPTIONS.labels('unexpected_guardian_address').inc()
+                return False
+
+            if message['stakingModuleId'] != module_id:
                 return False
 
             if message['blockNumber'] < latest['number'] - 200:

src/depositor.py

@@ -1,19 +1,18 @@
-from prometheus_client import start_http_server
 from flashbots import flashbot
+from prometheus_client import start_http_server
 from web3 import Web3
 from web3_multi_provider import FallbackProvider
 
 import variables
-from blockchain.executer import Executor
+from blockchain.executor import Executor
 from blockchain.web3_extentions.lido_contracts import LidoContracts
+from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware
 from blockchain.web3_extentions.transaction import TransactionUtils
 from bots.depositor import DepositorBot
 from metrics.healthcheck_pulse import start_pulse_server
 from metrics.logging import logging
-from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware
 from metrics.metrics import BUILD_INFO
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -67,7 +66,7 @@ def main():
         5,
         variables.MAX_CYCLE_LIFETIME_IN_SECONDS,
     )
-    logger.info({'msg': 'Rum executor.'})
+    logger.info({'msg': 'Execute depositor as daemon.'})
     e.execute_as_daemon()
 
 

src/metrics/metrics.py

@@ -26,13 +26,13 @@
 DEPOSIT_MESSAGES = Gauge(
     'deposit_messages',
     'Guardians deposit messages',
-    ['address', 'version'],
+    ['address', 'module_id', 'version'],
     namespace=PREFIX,
 )
 PAUSE_MESSAGES = Gauge(
     'pause_messages',
     'Guardians pause messages',
-    ['address', 'version'],
+    ['address', 'module_id', 'version'],
     namespace=PREFIX,
 )
 PING_MESSAGES = Gauge(
@@ -74,3 +74,10 @@
     ['method', 'code', 'domain'],
     namespace=PREFIX
 )
+
+UNEXPECTED_EXCEPTIONS = Counter(
+    'unexpected_exceptions',
+    'Total count of unexpected exceptions',
+    ['type'],
+    namespace=PREFIX,
+)

src/metrics/transport_message_metrics.py

@@ -4,7 +4,6 @@
 from transport.msg_providers.rabbit import MessageType
 from transport.msg_schemas import DepositMessage
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -15,11 +14,11 @@ def message_metrics_filter(msg: DepositMessage) -> bool:
     address, version = msg.get('guardianAddress'), msg.get('app', {}).get('version')
 
     if msg_type == MessageType.PAUSE:
-        PAUSE_MESSAGES.labels(address, version).inc()
+        PAUSE_MESSAGES.labels(address, msg.get('stakingModuleId', -1), version).inc()
         return True
 
     if msg_type == MessageType.DEPOSIT:
-        DEPOSIT_MESSAGES.labels(address, version).inc()
+        DEPOSIT_MESSAGES.labels(address, msg.get('stakingModuleId', -1), version).inc()
         return True
 
     elif msg_type == MessageType.PING:

src/pauser.py

@@ -3,16 +3,15 @@
 from web3_multi_provider import FallbackProvider
 
 import variables
-from blockchain.executer import Executor
+from blockchain.executor import Executor
 from blockchain.web3_extentions.lido_contracts import LidoContracts
+from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware
 from blockchain.web3_extentions.transaction import TransactionUtils
-from bots.pause import PauserBot
+from bots.pauser import PauserBot
 from metrics.healthcheck_pulse import start_pulse_server
 from metrics.logging import logging
-from blockchain.web3_extentions.requests_metric_middleware import add_requests_metric_middleware
 from metrics.metrics import BUILD_INFO
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -60,7 +59,7 @@ def main():
         1,
         variables.MAX_CYCLE_LIFETIME_IN_SECONDS,
     )
-    logger.info({'msg': 'Rum executor.'})
+    logger.info({'msg': 'Execute depositor as daemon.'})
     e.execute_as_daemon()
 
 

src/transport/msg_schemas.py

@@ -6,7 +6,7 @@
 from web3 import Web3
 
 from cryptography.verify_signature import verify_message_with_signature
-
+from metrics.metrics import UNEXPECTED_EXCEPTIONS
 
 logger = logging.getLogger(__name__)
 
@@ -74,11 +74,11 @@ class DepositMessage(TypedDict):
     app: dict
 
 
-def get_deposit_messages_sign_filter(attestation_prefix: bytes) -> Callable:
+def get_deposit_messages_sign_filter(deposit_prefix: bytes) -> Callable:
     """Returns filter that checks message validity"""
     def check_deposit_messages(msg: DepositMessage) -> bool:
         verified = verify_message_with_signature(
-            data=[attestation_prefix, msg['blockNumber'], msg['blockHash'], msg['depositRoot'], msg['stakingModuleId'], msg['nonce']],
+            data=[deposit_prefix, msg['blockNumber'], msg['blockHash'], msg['depositRoot'], msg['stakingModuleId'], msg['nonce']],
             abi=['bytes32', 'uint256', 'bytes32', 'bytes32', 'uint256', 'uint256'],
             address=msg['guardianAddress'],
             vrs=(
@@ -90,6 +90,7 @@ def check_deposit_messages(msg: DepositMessage) -> bool:
 
         if not verified:
             logger.error({'msg': 'Message verification failed.', 'value': msg})
+            UNEXPECTED_EXCEPTIONS.labels('deposit_message_verification_failed').inc()
 
         return verified
 
@@ -130,10 +131,10 @@ class PauseMessage(TypedDict):
     stakingModuleId: int
 
 
-def get_pause_messages_sign_filter(attestation_prefix: bytes) -> Callable:
+def get_pause_messages_sign_filter(pause_prefix: bytes) -> Callable:
     def check_pause_message(msg: PauseMessage) -> bool:
         verified = verify_message_with_signature(
-            data=[attestation_prefix, msg['blockNumber'], msg['stakingModuleId']],
+            data=[pause_prefix, msg['blockNumber'], msg['stakingModuleId']],
             abi=['bytes32', 'uint256', 'uint256'],
             address=msg['guardianAddress'],
             vrs=(
@@ -145,6 +146,7 @@ def check_pause_message(msg: PauseMessage) -> bool:
 
         if not verified:
             logger.error({'msg': 'Message verification failed.', 'value': msg})
+            UNEXPECTED_EXCEPTIONS.labels('pause_message_verification_failed').inc()
 
         return verified