diff --git a/Dockerfile b/Dockerfile index 87b71e1a0..e9c181d1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends -qq \ gcc=4:10.2.1-1 \ libffi-dev=3.3-6 \ g++=4:10.2.1-1 \ - git=1:2.30.2-1 \ curl=7.74.0-1.3+deb11u7 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index 5d4e3d1a9..ca7a641e2 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ In manual mode all sleeps are disabled and `ALLOW_REPORTING_IN_BUNKER_MODE` is T | `TX_GAS_ADDITION` | Used to modify gas parameter that used in transaction. (gas = estimated_gas + TX_GAS_ADDITION) | False | `100000` | | `CYCLE_SLEEP_IN_SECONDS` | The time between cycles of the oracle's activity | False | `12` | | `SUBMIT_DATA_DELAY_IN_SLOTS` | The difference in slots between submit data transactions from Oracles. It is used to prevent simultaneous sending of transactions and, as a result, transactions revert. | False | `6` | +| `HTTP_REQUEST_TIMEOUT_EXECUTION` | Timeout for HTTP execution layer requests | False | `120` | | `HTTP_REQUEST_TIMEOUT_CONSENSUS` | Timeout for HTTP consensus layer requests | False | `300` | | `HTTP_REQUEST_RETRY_COUNT_CONSENSUS` | Total number of retries to fetch data from endpoint for consensus layer requests | False | `5` | | `HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS` | The delay http provider sleeps if API is stuck for consensus layer | False | `12` | diff --git a/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_data_main_sleep_until_data_submitted.json b/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_data_main_sleep_until_data_submitted.json deleted file mode 100644 index 464e24e70..000000000 --- a/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_data_main_sleep_until_data_submitted.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "method": "eth_call", - "params": [ - { - "from": "0xF6d4bA61810778fF95BeA0B7DB2F103Dc042C5f7", - "to": "0x3FD30E8360e2E637be3428fB78A3d8D0ad157197", - "data": "0xfc7377cd00000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000001e000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000500000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000c" - }, - "latest" - ], - "response": { - "jsonrpc": "2.0", - "id": 0, - "error": { - "code": 3, - "message": "execution reverted", - "data": "0x23dada53" - } - } - } -] \ No newline at end of file diff --git a/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_submit_report.json b/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_submit_report.json deleted file mode 100644 index 464e24e70..000000000 --- a/fixtures/tests/modules/submodules/consensus/test_reports.py/test_process_report_submit_report.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "method": "eth_call", - "params": [ - { - "from": "0xF6d4bA61810778fF95BeA0B7DB2F103Dc042C5f7", - "to": "0x3FD30E8360e2E637be3428fB78A3d8D0ad157197", - "data": "0xfc7377cd00000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000001e000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000000d0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f0000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000500000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000c" - }, - "latest" - ], - "response": { - "jsonrpc": "2.0", - "id": 0, - "error": { - "code": 3, - "message": "execution reverted", - "data": "0x23dada53" - } - } - } -] \ No newline at end of file diff --git a/src/main.py b/src/main.py index a96b2e913..01be0bcbb 100644 --- a/src/main.py +++ b/src/main.py @@ -57,7 +57,10 @@ def main(module_name: OracleModule): start_http_server(variables.PROMETHEUS_PORT) logger.info({'msg': 'Initialize multi web3 provider.'}) - web3 = Web3(FallbackProviderModule(variables.EXECUTION_CLIENT_URI)) + web3 = Web3(FallbackProviderModule( + variables.EXECUTION_CLIENT_URI, + request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION} + )) logger.info({'msg': 'Modify web3 with custom contract function call.'}) tweak_w3_contracts(web3) diff --git a/src/modules/checks/suites/conftest.py b/src/modules/checks/suites/conftest.py index 06468749a..9e31852f3 100644 --- a/src/modules/checks/suites/conftest.py +++ b/src/modules/checks/suites/conftest.py @@ -1,6 +1,5 @@ import pytest from _pytest._io import TerminalWriter -from web3_multi_provider import MultiProvider from xdist import is_xdist_controller # type: ignore[import] from xdist.dsession import TerminalDistReporter # type: ignore[import] @@ -10,8 +9,12 @@ from src.utils.slot import get_reference_blockstamp from src.web3py.contract_tweak import tweak_w3_contracts from src.web3py.extensions import ( - ConsensusClientModule, KeysAPIClientModule, LidoValidatorsProvider, TransactionUtils, + ConsensusClientModule, + KeysAPIClientModule, + LidoValidatorsProvider, + TransactionUtils, LidoContracts, + FallbackProviderModule, ) from src.web3py.typings import Web3 @@ -21,7 +24,10 @@ @pytest.fixture() def web3(): - web3 = Web3(MultiProvider(variables.EXECUTION_CLIENT_URI)) + web3 = Web3(FallbackProviderModule( + variables.EXECUTION_CLIENT_URI, + request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION} + )) tweak_w3_contracts(web3) cc = ConsensusClientModule(variables.CONSENSUS_CLIENT_URI, web3) kac = KeysAPIClientModule(variables.KEYS_API_URI, web3) diff --git a/src/modules/submodules/consensus.py b/src/modules/submodules/consensus.py index 79cdc3f0a..2d8b002b5 100644 --- a/src/modules/submodules/consensus.py +++ b/src/modules/submodules/consensus.py @@ -270,6 +270,7 @@ def process_report(self, blockstamp: ReferenceBlockStamp) -> None: if not self.is_reporting_allowed(blockstamp): logger.warning({'msg': 'Reporting checks are not passed. Report will not be sent.'}) return + self._process_report_hash(blockstamp, report_hash) # Even if report hash transaction was failed we have to check if we can report data for current frame self._process_report_data(blockstamp, report_data, report_hash) @@ -277,22 +278,27 @@ def process_report(self, blockstamp: ReferenceBlockStamp) -> None: def _process_report_hash(self, blockstamp: ReferenceBlockStamp, report_hash: HexBytes) -> None: latest_blockstamp, member_info = self._get_latest_data() - # Check if current slot is newer than (member slot + slots_delay) - if not member_info.is_fast_lane: - if latest_blockstamp.slot_number < member_info.current_frame_ref_slot + member_info.fast_lane_length_slot: - logger.info({'msg': f'Member is not in fast lane, so report will be postponed for [{member_info.fast_lane_length_slot}] slots.'}) - return None - if not member_info.is_report_member: logger.info({'msg': 'Account can`t submit report hash.'}) return None - if HexBytes(member_info.current_frame_member_report) != report_hash: - logger.info({'msg': f'Send report hash. Consensus version: [{self.CONSENSUS_VERSION}]'}) - self._send_report_hash(blockstamp, report_hash, self.CONSENSUS_VERSION) - else: - logger.info({'msg': 'Provided hash already submitted.'}) + if HexBytes(member_info.current_frame_member_report) == report_hash: + logger.info({'msg': 'Account already submitted provided hash.'}) + return None + + if not member_info.is_fast_lane: + # Check if current slot is newer than (member slot + slots_delay) + if latest_blockstamp.slot_number < member_info.current_frame_ref_slot + member_info.fast_lane_length_slot: + logger.info({'msg': f'Member is not in fast lane, so report will be postponed ' + f'for [{member_info.fast_lane_length_slot}] slots.'}) + return None + + if HexBytes(member_info.current_frame_consensus_report) == report_hash: + logger.info({'msg': 'Consensus reached with provided hash.'}) + return None + logger.info({'msg': f'Send report hash. Consensus version: [{self.CONSENSUS_VERSION}]'}) + self._send_report_hash(blockstamp, report_hash, self.CONSENSUS_VERSION) return None def _process_report_data(self, blockstamp: ReferenceBlockStamp, report_data: tuple, report_hash: HexBytes): diff --git a/src/web3py/extensions/consistency.py b/src/providers/consistency.py similarity index 100% rename from src/web3py/extensions/consistency.py rename to src/providers/consistency.py diff --git a/src/providers/http_provider.py b/src/providers/http_provider.py index dbaa56630..9f97632cd 100644 --- a/src/providers/http_provider.py +++ b/src/providers/http_provider.py @@ -7,10 +7,9 @@ from prometheus_client import Histogram from requests import Session, JSONDecodeError from requests.adapters import HTTPAdapter -from requests.exceptions import ConnectionError as RequestsConnectionError from urllib3 import Retry -from src.web3py.extensions.consistency import ProviderConsistencyModule +from src.providers.consistency import ProviderConsistencyModule logger = logging.getLogger(__name__) @@ -126,8 +125,8 @@ def _get_without_fallbacks( params=query_params, timeout=self.request_timeout, ) - except RequestsConnectionError as error: - logger.debug({'msg': str(error)}) + except Exception as error: + logger.error({'msg': str(error)}) t.labels( endpoint=endpoint, code=0, diff --git a/src/variables.py b/src/variables.py index 3c7affd9e..1102aa679 100644 --- a/src/variables.py +++ b/src/variables.py @@ -51,6 +51,8 @@ CYCLE_SLEEP_IN_SECONDS = 0 # HTTP variables +HTTP_REQUEST_TIMEOUT_EXECUTION = int(os.getenv('HTTP_REQUEST_TIMEOUT_EXECUTION', 2 * 60)) + HTTP_REQUEST_TIMEOUT_CONSENSUS = int(os.getenv('HTTP_REQUEST_TIMEOUT_CONSENSUS', 5 * 60)) HTTP_REQUEST_RETRY_COUNT_CONSENSUS = int(os.getenv('HTTP_REQUEST_RETRY_COUNT_CONSENSUS', 5)) HTTP_REQUEST_SLEEP_BEFORE_RETRY_IN_SECONDS_CONSENSUS = int( diff --git a/src/web3py/extensions/__init__.py b/src/web3py/extensions/__init__.py index 2edd2242c..30382e166 100644 --- a/src/web3py/extensions/__init__.py +++ b/src/web3py/extensions/__init__.py @@ -4,4 +4,3 @@ from src.web3py.extensions.contracts import LidoContracts from src.web3py.extensions.lido_validators import LidoValidatorsProvider from src.web3py.extensions.fallback import FallbackProviderModule -from src.web3py.extensions.consistency import ProviderConsistencyModule diff --git a/src/web3py/extensions/fallback.py b/src/web3py/extensions/fallback.py index 3a8959d29..275762394 100644 --- a/src/web3py/extensions/fallback.py +++ b/src/web3py/extensions/fallback.py @@ -1,7 +1,7 @@ from typing import Any from web3_multi_provider import FallbackProvider -from src.web3py.extensions.consistency import ProviderConsistencyModule +from src.providers.consistency import ProviderConsistencyModule from web3 import Web3 diff --git a/tests/modules/submodules/consensus/test_reports.py b/tests/modules/submodules/consensus/test_reports.py index bda8145a1..bff035970 100644 --- a/tests/modules/submodules/consensus/test_reports.py +++ b/tests/modules/submodules/consensus/test_reports.py @@ -73,7 +73,11 @@ def test_report_hash(web3, consensus, tx_utils, set_report_account): def test_report_hash_member_not_in_fast_lane(web3, consensus, caplog): blockstamp = ReferenceBlockStampFactory.build() consensus._get_latest_blockstamp = Mock(return_value=blockstamp) - member_info = MemberInfoFactory.build(is_fast_lane=False, current_frame_ref_slot=blockstamp.slot_number - 1) + member_info = MemberInfoFactory.build( + is_fast_lane=False, + current_frame_ref_slot=blockstamp.slot_number - 1, + is_report_member=True, + ) consensus.get_member_info = Mock(return_value=member_info) consensus._process_report_hash(blockstamp, HexBytes(int.to_bytes(1, 32))) @@ -102,7 +106,7 @@ def test_do_not_report_same_hash(consensus, caplog, mock_latest_data): consensus.get_member_info = Mock(return_value=member_info) consensus._process_report_hash(blockstamp, HexBytes(int.to_bytes(1, 32))) - assert "Provided hash already submitted" in caplog.messages[-1] + assert "Account already submitted provided hash." in caplog.messages[-1] # -------- Process report data --------- @@ -140,6 +144,7 @@ def test_process_report_data_main_data_submitted(consensus, caplog, mock_latest_ blockstamp = ReferenceBlockStampFactory.build() report_data = tuple() report_hash = int.to_bytes(1, 32) + consensus._get_slot_delay_before_data_submit = Mock(return_value=0) consensus.is_main_data_submitted = Mock(side_effect=[False, True]) @@ -164,6 +169,7 @@ def test_process_report_data_main_sleep_until_data_submitted(consensus, caplog, consensus.is_main_data_submitted = Mock(return_value=False) consensus._get_slot_delay_before_data_submit = Mock(return_value=100) + consensus._submit_report = Mock() consensus._process_report_data(blockstamp, report_data, report_hash) assert "Sleep for 100 slots before sending data." in caplog.text @@ -203,6 +209,8 @@ def test_process_report_submit_report(consensus, tx_utils, caplog, mock_latest_d consensus.is_main_data_submitted = Mock(side_effect=main_data_submitted_base) consensus._get_slot_delay_before_data_submit = Mock(return_value=0) + consensus._submit_report = Mock() + consensus._process_report_data(blockstamp, report_data, report_hash) assert "Send report data. Contract version: [1]" in caplog.text